forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-16 8d2a02b24d66aa359e83eebc1ed3c0f85367a1cb 
 kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -33,7 +33,8 @@
3333 #include <mali_kbase_reset_gpu.h>
3434 #include <mali_kbase_ctx_sched.h>
3535 #include <mali_kbase_kinstr_jm.h>
36
-#include <mali_kbase_hwcnt_context.h>
36
+#include <mali_kbase_hwaccess_instr.h>
37
+#include <hwcnt/mali_kbase_hwcnt_context.h>
3738 #include <device/mali_kbase_device.h>
3839 #include <backend/gpu/mali_kbase_irq_internal.h>
3940 #include <backend/gpu/mali_kbase_jm_internal.h>
....@@ -43,23 +44,17 @@
4344 static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev,
4445 				const u64 affinity, const u64 limited_core_mask);
4546 
46
-static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
47
-				base_jd_core_req core_req,
48
-				int js, const u64 limited_core_mask)
47
+static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req core_req,
48
+				    unsigned int js, const u64 limited_core_mask)
4949 {
5050 	u64 affinity;
51
+	bool skip_affinity_check = false;
5152 
5253 	if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) ==
5354 			BASE_JD_REQ_T) {
54
-		/* Tiler-only atom */
55
-		/* If the hardware supports XAFFINITY then we'll only enable
56
-		 * the tiler (which is the default so this is a no-op),
57
-		 * otherwise enable shader core 0.
58
-		 */
59
-		if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY))
60
-			affinity = 1;
61
-		else
62
-			affinity = 0;
55
+		/* Tiler-only atom, affinity value can be programed as 0 */
56
+		affinity = 0;
57
+		skip_affinity_check = true;
6358 	} else if ((core_req & (BASE_JD_REQ_COHERENT_GROUP |
6459 			BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) {
6560 		unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
....@@ -89,7 +84,7 @@
8984 		affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask);
9085 	}
9186 
92
-	if (unlikely(!affinity)) {
87
+	if (unlikely(!affinity && !skip_affinity_check)) {
9388 #ifdef CONFIG_MALI_BIFROST_DEBUG
9489 		u64 shaders_ready =
9590 			kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
....@@ -195,22 +190,42 @@
195190 	return jc;
196191 }
197192 
198
-void kbase_job_hw_submit(struct kbase_device *kbdev,
199
-				struct kbase_jd_atom *katom,
200
-				int js)
193
+static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js,
194
+					  struct kbase_context *kctx)
195
+{
196
+	const ktime_t wait_loop_start = ktime_get_raw();
197
+	const s64 max_timeout = (s64)kbdev->js_data.js_free_wait_time_ms;
198
+	s64 diff = 0;
199
+
200
+	/* wait for the JS_COMMAND_NEXT register to reach the given status value */
201
+	do {
202
+		if (!kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)))
203
+			return true;
204
+
205
+		diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start));
206
+	} while (diff < max_timeout);
207
+
208
+	dev_err(kbdev->dev, "Timeout in waiting for job slot %u to become free for ctx %d_%u", js,
209
+		kctx->tgid, kctx->id);
210
+
211
+	return false;
212
+}
213
+
214
+int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js)
201215 {
202216 	struct kbase_context *kctx;
203217 	u32 cfg;
204218 	u64 const jc_head = select_job_chain(katom);
205219 	u64 affinity;
220
+	struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js];
206221 
207
-	KBASE_DEBUG_ASSERT(kbdev);
208
-	KBASE_DEBUG_ASSERT(katom);
222
+	lockdep_assert_held(&kbdev->hwaccess_lock);
209223 
210224 	kctx = katom->kctx;
211225 
212226 	/* Command register must be available */
213
-	KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
227
+	if (!kbasep_jm_wait_js_free(kbdev, js, kctx))
228
+		return -EPERM;
214229 
215230 	dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %pK\n",
216231 		jc_head, (void *)katom);
....@@ -232,9 +247,23 @@
232247 			!(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET))
233248 		cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
234249 
235
-	if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START))
236
-		cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
237
-	else
250
+	if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) {
251
+		/* Force a cache maintenance operation if the newly submitted
252
+		 * katom to the slot is from a different kctx. For a JM GPU
253
+		 * that has the feature BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER,
254
+		 * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a
255
+		 * FLUSH_CLEAN_INVALIDATE.
256
+		 */
257
+		u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged;
258
+
259
+		if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) {
260
+			if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER))
261
+				cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER;
262
+			else
263
+				cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
264
+		} else
265
+			cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
266
+	} else
238267 		cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
239268 
240269 	if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) &&
....@@ -251,18 +280,13 @@
251280 	    (katom->core_req & BASE_JD_REQ_END_RENDERPASS))
252281 		cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
253282 
254
-	if (kbase_hw_has_feature(kbdev,
255
-				BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
256
-		if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
257
-			cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
258
-			katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
259
-			kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
260
-								true;
261
-		} else {
262
-			katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
263
-			kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
264
-								false;
265
-		}
283
+	if (!ptr_slot_rb->job_chain_flag) {
284
+		cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
285
+		katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
286
+		ptr_slot_rb->job_chain_flag = true;
287
+	} else {
288
+		katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
289
+		ptr_slot_rb->job_chain_flag = false;
266290 	}
267291 
268292 	kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg);
....@@ -274,7 +298,7 @@
274298 	/* Write an approximate start timestamp.
275299 	 * It's approximate because there might be a job in the HEAD register.
276300 	 */
277
-	katom->start_timestamp = ktime_get();
301
+	katom->start_timestamp = ktime_get_raw();
278302 
279303 	/* GO ! */
280304 	dev_dbg(kbdev->dev, "JS: Submitting atom %pK from ctx %pK to js[%d] with head=0x%llx",
....@@ -300,6 +324,10 @@
300324 			&kbdev->gpu_props.props.raw_props.js_features[js],
301325 			"ctx_nr,atom_nr");
302326 	kbase_kinstr_jm_atom_hw_submit(katom);
327
+
328
+	/* Update the slot's last katom submission kctx */
329
+	ptr_slot_rb->last_kctx_tagged = SLOT_RB_TAG_KCTX(kctx);
330
+
303331 #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
304332 	if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
305333 		/* If this is the only job on the slot, trace it as starting */
....@@ -310,7 +338,6 @@
310338 						sizeof(js_string)),
311339 				ktime_to_ns(katom->start_timestamp),
312340 				(u32)katom->kctx->id, 0, katom->work_id);
313
-		kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
314341 	}
315342 #endif
316343 
....@@ -319,6 +346,8 @@
319346 
320347 	kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
321348 						JS_COMMAND_START);
349
+
350
+	return 0;
322351 }
323352 
324353 /**
....@@ -334,10 +363,8 @@
334363  * work out the best estimate (which might still result in an over-estimate to
335364  * the calculated time spent)
336365  */
337
-static void kbasep_job_slot_update_head_start_timestamp(
338
-						struct kbase_device *kbdev,
339
-						int js,
340
-						ktime_t end_timestamp)
366
+static void kbasep_job_slot_update_head_start_timestamp(struct kbase_device *kbdev, unsigned int js,
367
+							ktime_t end_timestamp)
341368 {
342369 	ktime_t timestamp_diff;
343370 	struct kbase_jd_atom *katom;
....@@ -367,8 +394,7 @@
367394  * Make a tracepoint call to the instrumentation module informing that
368395  * softstop happened on given lpu (job slot).
369396  */
370
-static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev,
371
-					int js)
397
+static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, unsigned int js)
372398 {
373399 	KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(
374400 		kbdev,
....@@ -377,19 +403,17 @@
377403 
378404 void kbase_job_done(struct kbase_device *kbdev, u32 done)
379405 {
380
-	int i;
381406 	u32 count = 0;
382407 	ktime_t end_timestamp;
383408 
384409 	lockdep_assert_held(&kbdev->hwaccess_lock);
385410 
386
-	KBASE_DEBUG_ASSERT(kbdev);
387
-
388411 	KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done);
389412 
390
-	end_timestamp = ktime_get();
413
+	end_timestamp = ktime_get_raw();
391414 
392415 	while (done) {
416
+		unsigned int i;
393417 		u32 failed = done >> 16;
394418 
395419 		/* treat failed slots as finished slots */
....@@ -399,7 +423,6 @@
399423 		 * numbered interrupts before the higher numbered ones.
400424 		 */
401425 		i = ffs(finished) - 1;
402
-		KBASE_DEBUG_ASSERT(i >= 0);
403426 
404427 		do {
405428 			int nr_done;
....@@ -415,6 +438,8 @@
415438 					JOB_SLOT_REG(i, JS_STATUS));
416439 
417440 				if (completion_code == BASE_JD_EVENT_STOPPED) {
441
+					u64 job_head;
442
+
418443 					KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(
419444 						kbdev, NULL,
420445 						i, 0, TL_JS_EVENT_SOFT_STOP);
....@@ -431,6 +456,21 @@
431456 						((u64)kbase_reg_read(kbdev,
432457 						JOB_SLOT_REG(i, JS_TAIL_HI))
433458 						 << 32);
459
+					job_head = (u64)kbase_reg_read(kbdev,
460
+						JOB_SLOT_REG(i, JS_HEAD_LO)) |
461
+						((u64)kbase_reg_read(kbdev,
462
+						JOB_SLOT_REG(i, JS_HEAD_HI))
463
+						 << 32);
464
+					/* For a soft-stopped job chain js_tail should
465
+					 * same as the js_head, but if not then the
466
+					 * job chain was incorrectly marked as
467
+					 * soft-stopped. In such case we should not
468
+					 * be resuming the job chain from js_tail and
469
+					 * report the completion_code as UNKNOWN.
470
+					 */
471
+					if (job_tail != job_head)
472
+						completion_code = BASE_JD_EVENT_UNKNOWN;
473
+
434474 				} else if (completion_code ==
435475 						BASE_JD_EVENT_NOT_STARTED) {
436476 					/* PRLAM-10673 can cause a TERMINATED
....@@ -563,7 +603,7 @@
563603 			failed = done >> 16;
564604 			finished = (done & 0xFFFF) | failed;
565605 			if (done)
566
-				end_timestamp = ktime_get();
606
+				end_timestamp = ktime_get_raw();
567607 		} while (finished & (1 << i));
568608 
569609 		kbasep_job_slot_update_head_start_timestamp(kbdev, i,
....@@ -581,18 +621,16 @@
581621 	KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
582622 }
583623 
584
-void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
585
-					int js,
586
-					u32 action,
587
-					base_jd_core_req core_reqs,
588
-					struct kbase_jd_atom *target_katom)
624
+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js,
625
+						 u32 action, base_jd_core_req core_reqs,
626
+						 struct kbase_jd_atom *target_katom)
589627 {
590628 #if KBASE_KTRACE_ENABLE
591629 	u32 status_reg_before;
592630 	u64 job_in_head_before;
593631 	u32 status_reg_after;
594632 
595
-	KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
633
+	WARN_ON(action & (~JS_COMMAND_MASK));
596634 
597635 	/* Check the head pointer */
598636 	job_in_head_before = ((u64) kbase_reg_read(kbdev,
....@@ -621,25 +659,17 @@
621659 		/* Mark the point where we issue the soft-stop command */
622660 		KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom);
623661 
624
-		if (kbase_hw_has_feature(
625
-				kbdev,
626
-				BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
627
-			action = (target_katom->atom_flags &
628
-					KBASE_KATOM_FLAGS_JOBCHAIN) ?
629
-				JS_COMMAND_SOFT_STOP_1 :
630
-				JS_COMMAND_SOFT_STOP_0;
631
-		}
662
+		action = (target_katom->atom_flags &
663
+			  KBASE_KATOM_FLAGS_JOBCHAIN) ?
664
+				 JS_COMMAND_SOFT_STOP_1 :
665
+				 JS_COMMAND_SOFT_STOP_0;
632666 	} else if (action == JS_COMMAND_HARD_STOP) {
633667 		target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
634668 
635
-		if (kbase_hw_has_feature(
636
-				kbdev,
637
-				BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
638
-			action = (target_katom->atom_flags &
639
-					KBASE_KATOM_FLAGS_JOBCHAIN) ?
640
-				JS_COMMAND_HARD_STOP_1 :
641
-				JS_COMMAND_HARD_STOP_0;
642
-		}
669
+		action = (target_katom->atom_flags &
670
+			  KBASE_KATOM_FLAGS_JOBCHAIN) ?
671
+				 JS_COMMAND_HARD_STOP_1 :
672
+				 JS_COMMAND_HARD_STOP_0;
643673 	}
644674 
645675 	kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action);
....@@ -651,6 +681,10 @@
651681 		struct kbase_context *head_kctx;
652682 
653683 		head = kbase_gpu_inspect(kbdev, js, 0);
684
+		if (unlikely(!head)) {
685
+			dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js);
686
+			return;
687
+		}
654688 		head_kctx = head->kctx;
655689 
656690 		if (status_reg_before == BASE_JD_EVENT_ACTIVE)
....@@ -678,7 +712,8 @@
678712 			KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
679713 			break;
680714 		default:
681
-			BUG();
715
+			WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action,
716
+			     (void *)target_katom, (void *)target_katom->kctx);
682717 			break;
683718 		}
684719 	} else {
....@@ -707,7 +742,8 @@
707742 			KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
708743 			break;
709744 		default:
710
-			BUG();
745
+			WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action,
746
+			     (void *)target_katom, (void *)target_katom->kctx);
711747 			break;
712748 		}
713749 	}
....@@ -717,7 +753,7 @@
717753 void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx)
718754 {
719755 	struct kbase_device *kbdev = kctx->kbdev;
720
-	int i;
756
+	unsigned int i;
721757 
722758 	lockdep_assert_held(&kbdev->hwaccess_lock);
723759 
....@@ -725,69 +761,33 @@
725761 		kbase_job_slot_hardstop(kctx, i, NULL);
726762 }
727763 
728
-/**
729
- * kbase_is_existing_atom_submitted_later_than_ready
730
- * @ready: sequence number of the ready atom
731
- * @existing: sequence number of the existing atom
732
- *
733
- * Returns true if the existing atom has been submitted later than the
734
- * ready atom. It is used to understand if an atom that is ready has been
735
- * submitted earlier than the currently running atom, so that the currently
736
- * running atom should be preempted to allow the ready atom to run.
737
- */
738
-static inline bool kbase_is_existing_atom_submitted_later_than_ready(u64 ready, u64 existing)
739
-{
740
-	/* No seq_nr set? */
741
-	if (!ready || !existing)
742
-		return false;
743
-
744
-	/* Efficiently handle the unlikely case of wrapping.
745
-	 * The following code assumes that the delta between the sequence number
746
-	 * of the two atoms is less than INT64_MAX.
747
-	 * In the extremely unlikely case where the delta is higher, the comparison
748
-	 * defaults for no preemption.
749
-	 * The code also assumes that the conversion from unsigned to signed types
750
-	 * works because the signed integers are 2's complement.
751
-	 */
752
-	return (s64)(ready - existing) < 0;
753
-}
754
-
755764 void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
756765 				struct kbase_jd_atom *target_katom)
757766 {
758767 	struct kbase_device *kbdev;
759
-	int js = target_katom->slot_nr;
760
-	int priority = target_katom->sched_priority;
761
-	int seq_nr = target_katom->seq_nr;
768
+	unsigned int target_js = target_katom->slot_nr;
762769 	int i;
763770 	bool stop_sent = false;
764771 
765
-	KBASE_DEBUG_ASSERT(kctx != NULL);
766772 	kbdev = kctx->kbdev;
767
-	KBASE_DEBUG_ASSERT(kbdev != NULL);
768773 
769774 	lockdep_assert_held(&kbdev->hwaccess_lock);
770775 
771
-	for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
772
-		struct kbase_jd_atom *katom;
776
+	for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, target_js); i++) {
777
+		struct kbase_jd_atom *slot_katom;
773778 
774
-		katom = kbase_gpu_inspect(kbdev, js, i);
775
-		if (!katom)
779
+		slot_katom = kbase_gpu_inspect(kbdev, target_js, i);
780
+		if (!slot_katom)
776781 			continue;
777782 
778
-		if ((kbdev->js_ctx_scheduling_mode ==
779
-			KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) &&
780
-				(katom->kctx != kctx))
781
-			continue;
782
-
783
-		if ((katom->sched_priority > priority) ||
784
-		    (katom->kctx == kctx && kbase_is_existing_atom_submitted_later_than_ready(seq_nr, katom->seq_nr))) {
783
+		if (kbase_js_atom_runs_before(kbdev, target_katom, slot_katom,
784
+					      KBASE_ATOM_ORDERING_FLAG_SEQNR)) {
785785 			if (!stop_sent)
786786 				KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED(
787787 						kbdev,
788788 						target_katom);
789789 
790
-			kbase_job_slot_softstop(kbdev, js, katom);
790
+			kbase_job_slot_softstop(kbdev, target_js, slot_katom);
791791 			stop_sent = true;
792792 		}
793793 	}
....@@ -875,7 +875,7 @@
875875 	if (timeout != 0)
876876 		goto exit;
877877 
878
-	if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) {
878
+	if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) {
879879 		dev_err(kbdev->dev,
880880 			"Issuing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
881881 			ZAP_TIMEOUT);
....@@ -943,46 +943,29 @@
943943  *
944944  * Where possible any job in the next register is evicted before the soft-stop.
945945  */
946
-void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
947
-			struct kbase_jd_atom *target_katom, u32 sw_flags)
946
+void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js,
947
+				     struct kbase_jd_atom *target_katom, u32 sw_flags)
948948 {
949949 	dev_dbg(kbdev->dev, "Soft-stop atom %pK with flags 0x%x (s:%d)\n",
950950 		target_katom, sw_flags, js);
951951 
952
-	KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
952
+	if (sw_flags & JS_COMMAND_MASK) {
953
+		WARN(true, "Atom %pK in kctx %pK received non-NOP flags %d\n", (void *)target_katom,
954
+		     target_katom ? (void *)target_katom->kctx : NULL, sw_flags);
955
+		sw_flags &= ~((u32)JS_COMMAND_MASK);
956
+	}
953957 	kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
954958 			JS_COMMAND_SOFT_STOP | sw_flags);
955959 }
956960 
957
-/**
958
- * kbase_job_slot_softstop - Soft-stop the specified job slot
959
- * @kbdev:         The kbase device
960
- * @js:            The job slot to soft-stop
961
- * @target_katom:  The job that should be soft-stopped (or NULL for any job)
962
- * Context:
963
- *   The job slot lock must be held when calling this function.
964
- *   The job slot must not already be in the process of being soft-stopped.
965
- *
966
- * Where possible any job in the next register is evicted before the soft-stop.
967
- */
968961 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
969962 				struct kbase_jd_atom *target_katom)
970963 {
971964 	kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
972965 }
973966 
974
-/**
975
- * kbase_job_slot_hardstop - Hard-stop the specified job slot
976
- * @kctx:         The kbase context that contains the job(s) that should
977
- *                be hard-stopped
978
- * @js:           The job slot to hard-stop
979
- * @target_katom: The job that should be hard-stopped (or NULL for all
980
- *                jobs from the context)
981
- * Context:
982
- *   The job slot lock must be held when calling this function.
983
- */
984
-void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
985
-				struct kbase_jd_atom *target_katom)
967
+void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js,
968
+			     struct kbase_jd_atom *target_katom)
986969 {
987970 	struct kbase_device *kbdev = kctx->kbdev;
988971 	bool stopped;
....@@ -990,28 +973,9 @@
990973 	stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
991974 							target_katom,
992975 							JS_COMMAND_HARD_STOP);
976
+	CSTD_UNUSED(stopped);
993977 }
994978 
995
-/**
996
- * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
997
- * @kbdev: kbase device
998
- * @action: the event which has occurred
999
- * @core_reqs: core requirements of the atom
1000
- * @target_katom: the atom which is being affected
1001
- *
1002
- * For a certain soft-stop action, work out whether to enter disjoint
1003
- * state.
1004
- *
1005
- * This does not register multiple disjoint events if the atom has already
1006
- * started a disjoint period
1007
- *
1008
- * @core_reqs can be supplied as 0 if the atom had not started on the hardware
1009
- * (and so a 'real' soft/hard-stop was not required, but it still interrupted
1010
- * flow, perhaps on another context)
1011
- *
1012
- * kbase_job_check_leave_disjoint() should be used to end the disjoint
1013
- * state when the soft/hard-stop action is complete
1014
- */
1015979 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
1016980 		base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom)
1017981 {
....@@ -1033,14 +997,6 @@
1033997 	kbase_disjoint_state_up(kbdev);
1034998 }
1035999 
1036
-/**
1037
- * kbase_job_check_enter_disjoint - potentially leave disjoint state
1038
- * @kbdev: kbase device
1039
- * @target_katom: atom which is finishing
1040
- *
1041
- * Work out whether to leave disjoint state when finishing an atom that was
1042
- * originated by kbase_job_check_enter_disjoint().
1043
- */
10441000 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
10451001 		struct kbase_jd_atom *target_katom)
10461002 {
....@@ -1117,21 +1073,21 @@
11171073 {
11181074 	unsigned long flags;
11191075 	struct kbase_device *kbdev;
1120
-	ktime_t end_timestamp = ktime_get();
1076
+	ktime_t end_timestamp = ktime_get_raw();
11211077 	struct kbasep_js_device_data *js_devdata;
11221078 	bool silent = false;
11231079 	u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
11241080 
1125
-	KBASE_DEBUG_ASSERT(data);
1126
-
11271081 	kbdev = container_of(data, struct kbase_device,
11281082 						hwaccess.backend.reset_work);
11291083 
1130
-	KBASE_DEBUG_ASSERT(kbdev);
11311084 	js_devdata = &kbdev->js_data;
11321085 
11331086 	if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
11341087 			KBASE_RESET_GPU_SILENT)
1088
+		silent = true;
1089
+
1090
+	if (kbase_is_quick_reset_enabled(kbdev))
11351091 		silent = true;
11361092 
11371093 	KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
....@@ -1162,7 +1118,7 @@
11621118 		return;
11631119 	}
11641120 
1165
-	KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false);
1121
+	WARN(kbdev->irq_reset_flush, "%s: GPU reset already in flight\n", __func__);
11661122 
11671123 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
11681124 	spin_lock(&kbdev->mmu_mask_change);
....@@ -1203,7 +1159,8 @@
12031159 
12041160 	mutex_lock(&kbdev->pm.lock);
12051161 	/* We hold the pm lock, so there ought to be a current policy */
1206
-	KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
1162
+	if (unlikely(!kbdev->pm.backend.pm_current_policy))
1163
+		dev_warn(kbdev->dev, "No power policy set!");
12071164 
12081165 	/* All slot have been soft-stopped and we've waited
12091166 	 * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
....@@ -1228,6 +1185,13 @@
12281185 		kbase_backend_reset(kbdev, &end_timestamp);
12291186 	kbase_pm_metrics_update(kbdev, NULL);
12301187 	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1188
+
1189
+	/* Tell hardware counters a reset is about to occur.
1190
+	 * If the instr backend is in an unrecoverable error state (e.g. due to
1191
+	 * HW being unresponsive), this will transition the backend out of
1192
+	 * it, on the assumption a reset will fix whatever problem there was.
1193
+	 */
1194
+	kbase_instr_hwcnt_on_before_reset(kbdev);
12311195 
12321196 	/* Reset the GPU */
12331197 	kbase_pm_init_hw(kbdev, 0);
....@@ -1293,8 +1257,6 @@
12931257 	struct kbase_device *kbdev = container_of(timer, struct kbase_device,
12941258 						hwaccess.backend.reset_timer);
12951259 
1296
-	KBASE_DEBUG_ASSERT(kbdev);
1297
-
12981260 	/* Reset still pending? */
12991261 	if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
13001262 			KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
....@@ -1312,10 +1274,8 @@
13121274 
13131275 static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
13141276 {
1315
-	int i;
1277
+	unsigned int i;
13161278 	int pending_jobs = 0;
1317
-
1318
-	KBASE_DEBUG_ASSERT(kbdev);
13191279 
13201280 	/* Count the number of jobs */
13211281 	for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
....@@ -1361,11 +1321,10 @@
13611321  * @kbdev: kbase device
13621322  * @flags: Bitfield indicating impact of reset (see flag defines)
13631323  *
1364
- * This function just soft-stops all the slots to ensure that as many jobs as
1324
+ * This function soft-stops all the slots to ensure that as many jobs as
13651325  * possible are saved.
13661326  *
1367
- * Return:
1368
- *   The function returns a boolean which should be interpreted as follows:
1327
+ * Return: boolean which should be interpreted as follows:
13691328  *   true - Prepared for reset, kbase_reset_gpu_locked should be called.
13701329  *   false - Another thread is performing a reset, kbase_reset_gpu should
13711330  *   not be called.
....@@ -1375,9 +1334,6 @@
13751334 {
13761335 	int i;
13771336 
1378
-	CSTD_UNUSED(flags);
1379
-	KBASE_DEBUG_ASSERT(kbdev);
1380
-
13811337 #ifdef CONFIG_MALI_ARBITER_SUPPORT
13821338 	if (kbase_pm_is_gpu_lost(kbdev)) {
13831339 		/* GPU access has been removed, reset will be done by
....@@ -1386,6 +1342,9 @@
13861342 		return false;
13871343 	}
13881344 #endif
1345
+
1346
+	if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)
1347
+		kbase_instr_hwcnt_on_unrecoverable_error(kbdev);
13891348 
13901349 	if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
13911350 						KBASE_RESET_GPU_NOT_PENDING,
....@@ -1428,18 +1387,17 @@
14281387  */
14291388 void kbase_reset_gpu(struct kbase_device *kbdev)
14301389 {
1431
-	KBASE_DEBUG_ASSERT(kbdev);
1432
-
14331390 	/* Note this is an assert/atomic_set because it is a software issue for
14341391 	 * a race to be occurring here
14351392 	 */
1436
-	KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1437
-						KBASE_RESET_GPU_PREPARED);
1393
+	if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED))
1394
+		return;
14381395 	atomic_set(&kbdev->hwaccess.backend.reset_gpu,
14391396 						KBASE_RESET_GPU_COMMITTED);
14401397 
1441
-	dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1442
-			kbdev->reset_timeout_ms);
1398
+	if (!kbase_is_quick_reset_enabled(kbdev))
1399
+		dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1400
+				kbdev->reset_timeout_ms);
14431401 
14441402 	hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
14451403 			HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
....@@ -1452,18 +1410,17 @@
14521410 
14531411 void kbase_reset_gpu_locked(struct kbase_device *kbdev)
14541412 {
1455
-	KBASE_DEBUG_ASSERT(kbdev);
1456
-
14571413 	/* Note this is an assert/atomic_set because it is a software issue for
14581414 	 * a race to be occurring here
14591415 	 */
1460
-	KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1461
-						KBASE_RESET_GPU_PREPARED);
1416
+	if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED))
1417
+		return;
14621418 	atomic_set(&kbdev->hwaccess.backend.reset_gpu,
14631419 						KBASE_RESET_GPU_COMMITTED);
14641420 
1465
-	dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1466
-			kbdev->reset_timeout_ms);
1421
+	if (!kbase_is_quick_reset_enabled(kbdev))
1422
+		dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1423
+				kbdev->reset_timeout_ms);
14671424 	hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
14681425 			HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
14691426 			HRTIMER_MODE_REL);
....@@ -1497,6 +1454,11 @@
14971454 		return false;
14981455 
14991456 	return true;
1457
+}
1458
+
1459
+bool kbase_reset_gpu_is_not_pending(struct kbase_device *kbdev)
1460
+{
1461
+	return atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING;
15001462 }
15011463 
15021464 int kbase_reset_gpu_wait(struct kbase_device *kbdev)
....@@ -1540,9 +1502,9 @@
15401502 #ifdef CONFIG_MALI_BIFROST_DEBUG
15411503 	dev_dbg(kbdev->dev,
15421504 				"Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK from 0x%lx to 0x%lx (mask is 0x%lx)\n",
1543
-				(unsigned long int)affinity,
1544
-				(unsigned long int)result,
1545
-				(unsigned long int)limited_core_mask);
1505
+				(unsigned long)affinity,
1506
+				(unsigned long)result,
1507
+				(unsigned long)limited_core_mask);
15461508 #else
15471509 	CSTD_UNUSED(kbdev);
15481510 #endif