~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,7 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
2	2	/*
3	3	*
4		- * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved.
	4	+ * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved.
5	5	*
6	6	* This program is free software and is provided to you under the terms of the
7	7	* GNU General Public License version 2 as published by the Free Software
..	..	@@ -34,6 +34,7 @@
34	34
35	35	#include "mali_kbase_jm.h"
36	36	#include "mali_kbase_hwaccess_jm.h"
	37	+#include <mali_kbase_hwaccess_time.h>
37	38	#include <linux/priority_control_manager.h>
38	39
39	40	/*
..	..	@@ -77,8 +78,7 @@
77	78	struct kbase_device kbdev, struct kbase_context kctx,
78	79	struct kbasep_js_atom_retained_state *katom_retained_state);
79	80
80		-static int kbase_js_get_slot(struct kbase_device *kbdev,
81		- struct kbase_jd_atom *katom);
	81	+static unsigned int kbase_js_get_slot(struct kbase_device kbdev, struct kbase_jd_atom katom);
82	82
83	83	static void kbase_js_foreach_ctx_job(struct kbase_context *kctx,
84	84	kbasep_js_ctx_job_cb *callback);
..	..	@@ -151,8 +151,7 @@
151	151	*
152	152	* Return: true if there are no atoms to pull, false otherwise.
153	153	*/
154		-static inline bool
155		-jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio)
	154	+static inline bool jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, unsigned int js, int prio)
156	155	{
157	156	bool none_to_pull;
158	157	struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
..	..	@@ -161,9 +160,8 @@
161	160
162	161	none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree);
163	162
164		- dev_dbg(kctx->kbdev->dev,
165		- "Slot %d (prio %d) is %spullable in kctx %pK\n",
166		- js, prio, none_to_pull ? "not " : "", kctx);
	163	+ dev_dbg(kctx->kbdev->dev, "Slot %u (prio %d) is %spullable in kctx %pK\n", js, prio,
	164	+ none_to_pull ? "not " : "", kctx);
167	165
168	166	return none_to_pull;
169	167	}
..	..	@@ -179,8 +177,7 @@
179	177	* Return: true if the ring buffers for all priorities have no pullable atoms,
180	178	* false otherwise.
181	179	*/
182		-static inline bool
183		-jsctx_rb_none_to_pull(struct kbase_context *kctx, int js)
	180	+static inline bool jsctx_rb_none_to_pull(struct kbase_context *kctx, unsigned int js)
184	181	{
185	182	int prio;
186	183
..	..	@@ -212,8 +209,8 @@
212	209	*
213	210	* The HW access lock must always be held when calling this function.
214	211	*/
215		-static void jsctx_queue_foreach_prio(struct kbase_context *kctx, int js,
216		- int prio, kbasep_js_ctx_job_cb *callback)
	212	+static void jsctx_queue_foreach_prio(struct kbase_context *kctx, unsigned int js, int prio,
	213	+ kbasep_js_ctx_job_cb *callback)
217	214	{
218	215	struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
219	216
..	..	@@ -272,7 +269,7 @@
272	269	* jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback
273	270	* for each entry, and remove the entry from the queue.
274	271	*/
275		-static inline void jsctx_queue_foreach(struct kbase_context *kctx, int js,
	272	+static inline void jsctx_queue_foreach(struct kbase_context *kctx, unsigned int js,
276	273	kbasep_js_ctx_job_cb *callback)
277	274	{
278	275	int prio;
..	..	@@ -293,15 +290,14 @@
293	290	*
294	291	* Return: Pointer to next atom in buffer, or NULL if there is no atom.
295	292	*/
296		-static inline struct kbase_jd_atom *
297		-jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio)
	293	+static inline struct kbase_jd_atom jsctx_rb_peek_prio(struct kbase_context kctx, unsigned int js,
	294	+ int prio)
298	295	{
299	296	struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
300	297	struct rb_node *node;
301	298
302	299	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
303		- dev_dbg(kctx->kbdev->dev,
304		- "Peeking runnable tree of kctx %pK for prio %d (s:%d)\n",
	300	+ dev_dbg(kctx->kbdev->dev, "Peeking runnable tree of kctx %pK for prio %d (s:%u)\n",
305	301	(void *)kctx, prio, js);
306	302
307	303	node = rb_first(&rb->runnable_tree);
..	..	@@ -326,8 +322,7 @@
326	322	*
327	323	* Return: Pointer to next atom in buffer, or NULL if there is no atom.
328	324	*/
329		-static inline struct kbase_jd_atom *
330		-jsctx_rb_peek(struct kbase_context *kctx, int js)
	325	+static inline struct kbase_jd_atom jsctx_rb_peek(struct kbase_context kctx, unsigned int js)
331	326	{
332	327	int prio;
333	328
..	..	@@ -358,7 +353,7 @@
358	353	jsctx_rb_pull(struct kbase_context kctx, struct kbase_jd_atom katom)
359	354	{
360	355	int prio = katom->sched_priority;
361		- int js = katom->slot_nr;
	356	+ unsigned int js = katom->slot_nr;
362	357	struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js];
363	358
364	359	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
..	..	@@ -372,28 +367,26 @@
372	367	rb_erase(&katom->runnable_tree_node, &rb->runnable_tree);
373	368	}
374	369
375		-#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0)
376		-
377	370	static void
378	371	jsctx_tree_add(struct kbase_context kctx, struct kbase_jd_atom katom)
379	372	{
380	373	struct kbase_device *kbdev = kctx->kbdev;
381	374	int prio = katom->sched_priority;
382		- int js = katom->slot_nr;
	375	+ unsigned int js = katom->slot_nr;
383	376	struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
384	377	struct rb_node *new = &(queue->runnable_tree.rb_node), parent = NULL;
385	378
386	379	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
387	380
388		- dev_dbg(kbdev->dev, "Adding atom %pK to runnable tree of kctx %pK (s:%d)\n",
389		- (void )katom, (void )kctx, js);
	381	+ dev_dbg(kbdev->dev, "Adding atom %pK to runnable tree of kctx %pK (s:%u)\n", (void *)katom,
	382	+ (void *)kctx, js);
390	383
391	384	while (*new) {
392	385	struct kbase_jd_atom entry = container_of(new,
393	386	struct kbase_jd_atom, runnable_tree_node);
394	387
395	388	parent = *new;
396		- if (LESS_THAN_WRAP(katom->age, entry->age))
	389	+ if (kbase_jd_atom_is_younger(katom, entry))
397	390	new = &((*new)->rb_left);
398	391	else
399	392	new = &((*new)->rb_right);
..	..	@@ -421,18 +414,78 @@
421	414	{
422	415	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
423	416
	417	+ KBASE_KTRACE_ADD_JM(kctx->kbdev, JS_UNPULL_JOB, kctx, katom, katom->jc,
	418	+ 0u);
	419	+
424	420	jsctx_tree_add(kctx, katom);
425	421	}
426	422
427		-static bool kbase_js_ctx_pullable(struct kbase_context *kctx,
428		- int js,
429		- bool is_scheduled);
	423	+static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled);
430	424	static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev,
431		- struct kbase_context *kctx,
432		- int js);
	425	+ struct kbase_context *kctx, unsigned int js);
433	426	static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev,
434		- struct kbase_context *kctx,
435		- int js);
	427	+ struct kbase_context *kctx, unsigned int js);
	428	+
	429	+typedef bool(katom_ordering_func)(const struct kbase_jd_atom *,
	430	+ const struct kbase_jd_atom *);
	431	+
	432	+bool kbase_js_atom_runs_before(struct kbase_device *kbdev,
	433	+ const struct kbase_jd_atom *katom_a,
	434	+ const struct kbase_jd_atom *katom_b,
	435	+ const kbase_atom_ordering_flag_t order_flags)
	436	+{
	437	+ struct kbase_context *kctx_a = katom_a->kctx;
	438	+ struct kbase_context *kctx_b = katom_b->kctx;
	439	+ katom_ordering_func *samectxatomprio_ordering_func =
	440	+ kbase_jd_atom_is_younger;
	441	+
	442	+ lockdep_assert_held(&kbdev->hwaccess_lock);
	443	+
	444	+ if (order_flags & KBASE_ATOM_ORDERING_FLAG_SEQNR)
	445	+ samectxatomprio_ordering_func = kbase_jd_atom_is_earlier;
	446	+
	447	+ /* It only makes sense to make this test for atoms on the same slot */
	448	+ WARN_ON(katom_a->slot_nr != katom_b->slot_nr);
	449	+
	450	+ if (kbdev->js_ctx_scheduling_mode ==
	451	+ KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) {
	452	+ /* In local priority mode, querying either way around for "a
	453	+ * should run before b" and "b should run before a" should
	454	+ * always be false when they're from different contexts
	455	+ */
	456	+ if (kctx_a != kctx_b)
	457	+ return false;
	458	+ } else {
	459	+ /* In system priority mode, ordering is done first strictly by
	460	+ * context priority, even when katom_b might be lower priority
	461	+ * than katom_a. This is due to scheduling of contexts in order
	462	+ * of highest priority first, regardless of whether the atoms
	463	+ * for a particular slot from such contexts have the highest
	464	+ * priority or not.
	465	+ */
	466	+ if (kctx_a != kctx_b) {
	467	+ if (kctx_a->priority < kctx_b->priority)
	468	+ return true;
	469	+ if (kctx_a->priority > kctx_b->priority)
	470	+ return false;
	471	+ }
	472	+ }
	473	+
	474	+ /* For same contexts/contexts with the same context priority (in system
	475	+ * priority mode), ordering is next done by atom priority
	476	+ */
	477	+ if (katom_a->sched_priority < katom_b->sched_priority)
	478	+ return true;
	479	+ if (katom_a->sched_priority > katom_b->sched_priority)
	480	+ return false;
	481	+ /* For atoms of same priority on the same kctx, they are
	482	+ * ordered by seq_nr/age (dependent on caller)
	483	+ */
	484	+ if (kctx_a == kctx_b && samectxatomprio_ordering_func(katom_a, katom_b))
	485	+ return true;
	486	+
	487	+ return false;
	488	+}
436	489
437	490	/*
438	491	* Functions private to KBase ('Protected' functions)
..	..	@@ -475,9 +528,11 @@
475	528	jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING;
476	529	jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS;
477	530	jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL;
	531	+
478	532	jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING;
479	533	jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS;
480	534	atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT);
	535	+ jsdd->js_free_wait_time_ms = kbase_get_timeout_ms(kbdev, JM_DEFAULT_JS_FREE_TIMEOUT);
481	536
482	537	dev_dbg(kbdev->dev, "JS Config Attribs: ");
483	538	dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u",
..	..	@@ -502,6 +557,7 @@
502	557	jsdd->ctx_timeslice_ns);
503	558	dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i",
504	559	atomic_read(&jsdd->soft_job_timeout_ms));
	560	+ dev_dbg(kbdev->dev, "\tjs_free_wait_time_ms:%u", jsdd->js_free_wait_time_ms);
505	561
506	562	if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss &&
507	563	jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss &&
..	..	@@ -558,6 +614,7 @@
558	614	{
559	615	struct kbasep_js_device_data *js_devdata;
560	616	s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, };
	617	+ CSTD_UNUSED(js_devdata);
561	618
562	619	KBASE_DEBUG_ASSERT(kbdev != NULL);
563	620
..	..	@@ -575,14 +632,13 @@
575	632
576	633	int kbasep_js_kctx_init(struct kbase_context *const kctx)
577	634	{
578		- struct kbase_device *kbdev;
579	635	struct kbasep_js_kctx_info *js_kctx_info;
580	636	int i, j;
	637	+ CSTD_UNUSED(js_kctx_info);
581	638
582	639	KBASE_DEBUG_ASSERT(kctx != NULL);
583	640
584		- kbdev = kctx->kbdev;
585		- KBASE_DEBUG_ASSERT(kbdev != NULL);
	641	+ kbase_ctx_sched_init_ctx(kctx);
586	642
587	643	for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i)
588	644	INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]);
..	..	@@ -622,9 +678,10 @@
622	678	{
623	679	struct kbase_device *kbdev;
624	680	struct kbasep_js_kctx_info *js_kctx_info;
625		- int js;
	681	+ unsigned int js;
626	682	bool update_ctx_count = false;
627	683	unsigned long flags;
	684	+ CSTD_UNUSED(js_kctx_info);
628	685
629	686	KBASE_DEBUG_ASSERT(kctx != NULL);
630	687
..	..	@@ -660,6 +717,149 @@
660	717	kbase_backend_ctx_count_changed(kbdev);
661	718	mutex_unlock(&kbdev->js_data.runpool_mutex);
662	719	}
	720	+
	721	+ kbase_ctx_sched_remove_ctx(kctx);
	722	+}
	723	+
	724	+/*
	725	+ * Priority blocking management functions
	726	+ */
	727	+
	728	+/* Should not normally use directly - use kbase_jsctx_slot_atom_pulled_dec() instead */
	729	+static void kbase_jsctx_slot_prio_blocked_clear(struct kbase_context *kctx, unsigned int js,
	730	+ int sched_prio)
	731	+{
	732	+ struct kbase_jsctx_slot_tracking *slot_tracking =
	733	+ &kctx->slot_tracking[js];
	734	+
	735	+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
	736	+
	737	+ slot_tracking->blocked &= ~(((kbase_js_prio_bitmap_t)1) << sched_prio);
	738	+ KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_UNBLOCKED, kctx,
	739	+ NULL, 0, js, (unsigned int)sched_prio);
	740	+}
	741	+
	742	+static int kbase_jsctx_slot_atoms_pulled(struct kbase_context *kctx, unsigned int js)
	743	+{
	744	+ return atomic_read(&kctx->slot_tracking[js].atoms_pulled);
	745	+}
	746	+
	747	+/*
	748	+ * A priority level on a slot is blocked when:
	749	+ * - that priority level is blocked
	750	+ * - or, any higher priority level is blocked
	751	+ */
	752	+static bool kbase_jsctx_slot_prio_is_blocked(struct kbase_context *kctx, unsigned int js,
	753	+ int sched_prio)
	754	+{
	755	+ struct kbase_jsctx_slot_tracking *slot_tracking =
	756	+ &kctx->slot_tracking[js];
	757	+ kbase_js_prio_bitmap_t prio_bit, higher_prios_mask;
	758	+
	759	+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
	760	+
	761	+ /* done in two separate shifts to prevent future undefined behavior
	762	+ * should the number of priority levels == (bit width of the type)
	763	+ */
	764	+ prio_bit = (((kbase_js_prio_bitmap_t)1) << sched_prio);
	765	+ /* all bits of sched_prio or higher, with sched_prio = 0 being the
	766	+ * highest priority
	767	+ */
	768	+ higher_prios_mask = (prio_bit << 1) - 1u;
	769	+ return (slot_tracking->blocked & higher_prios_mask) != 0u;
	770	+}
	771	+
	772	+/**
	773	+ * kbase_jsctx_slot_atom_pulled_inc - Increase counts of atoms that have being
	774	+ * pulled for a slot from a ctx, based on
	775	+ * this atom
	776	+ * @kctx: kbase context
	777	+ * @katom: atom pulled
	778	+ *
	779	+ * Manages counts of atoms pulled (including per-priority-level counts), for
	780	+ * later determining when a ctx can become unblocked on a slot.
	781	+ *
	782	+ * Once a slot has been blocked at @katom's priority level, it should not be
	783	+ * pulled from, hence this function should not be called in that case.
	784	+ *
	785	+ * The return value is to aid tracking of when @kctx becomes runnable.
	786	+ *
	787	+ * Return: new total count of atoms pulled from all slots on @kctx
	788	+ */
	789	+static int kbase_jsctx_slot_atom_pulled_inc(struct kbase_context *kctx,
	790	+ const struct kbase_jd_atom *katom)
	791	+{
	792	+ unsigned int js = katom->slot_nr;
	793	+ int sched_prio = katom->sched_priority;
	794	+ struct kbase_jsctx_slot_tracking *slot_tracking =
	795	+ &kctx->slot_tracking[js];
	796	+ int nr_atoms_pulled;
	797	+
	798	+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
	799	+
	800	+ WARN(kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio),
	801	+ "Should not have pulled atoms for slot %u from a context that is blocked at priority %d or higher",
	802	+ js, sched_prio);
	803	+
	804	+ nr_atoms_pulled = atomic_inc_return(&kctx->atoms_pulled_all_slots);
	805	+ atomic_inc(&slot_tracking->atoms_pulled);
	806	+ slot_tracking->atoms_pulled_pri[sched_prio]++;
	807	+
	808	+ return nr_atoms_pulled;
	809	+}
	810	+
	811	+/**
	812	+ * kbase_jsctx_slot_atom_pulled_dec- Decrease counts of atoms that have being
	813	+ * pulled for a slot from a ctx, and
	814	+ * re-evaluate whether a context is blocked
	815	+ * on this slot
	816	+ * @kctx: kbase context
	817	+ * @katom: atom that has just been removed from a job slot
	818	+ *
	819	+ * @kctx can become unblocked on a slot for a priority level when it no longer
	820	+ * has any pulled atoms at that priority level on that slot, and all higher
	821	+ * (numerically lower) priority levels are also unblocked @kctx on that
	822	+ * slot. The latter condition is to retain priority ordering within @kctx.
	823	+ *
	824	+ * Return: true if the slot was previously blocked but has now become unblocked
	825	+ * at @katom's priority level, false otherwise.
	826	+ */
	827	+static bool kbase_jsctx_slot_atom_pulled_dec(struct kbase_context *kctx,
	828	+ const struct kbase_jd_atom *katom)
	829	+{
	830	+ unsigned int js = katom->slot_nr;
	831	+ int sched_prio = katom->sched_priority;
	832	+ int atoms_pulled_pri;
	833	+ struct kbase_jsctx_slot_tracking *slot_tracking =
	834	+ &kctx->slot_tracking[js];
	835	+ bool slot_prio_became_unblocked = false;
	836	+
	837	+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
	838	+
	839	+ atomic_dec(&kctx->atoms_pulled_all_slots);
	840	+ atomic_dec(&slot_tracking->atoms_pulled);
	841	+
	842	+ atoms_pulled_pri = --(slot_tracking->atoms_pulled_pri[sched_prio]);
	843	+
	844	+ /* We can safely clear this priority level's blocked status even if
	845	+ * higher priority levels are still blocked: a subsequent query to
	846	+ * kbase_jsctx_slot_prio_is_blocked() will still return true
	847	+ */
	848	+ if (!atoms_pulled_pri &&
	849	+ kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio)) {
	850	+ kbase_jsctx_slot_prio_blocked_clear(kctx, js, sched_prio);
	851	+
	852	+ if (!kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio))
	853	+ slot_prio_became_unblocked = true;
	854	+ }
	855	+
	856	+ if (slot_prio_became_unblocked)
	857	+ KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev,
	858	+ JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED,
	859	+ kctx, katom, katom->jc, js,
	860	+ (unsigned int)sched_prio);
	861	+
	862	+ return slot_prio_became_unblocked;
663	863	}
664	864
665	865	/**
..	..	@@ -676,14 +876,12 @@
676	876	* Return: true if caller should call kbase_backend_ctx_count_changed()
677	877	*/
678	878	static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev,
679		- struct kbase_context *kctx,
680		- int js)
	879	+ struct kbase_context *kctx, unsigned int js)
681	880	{
682	881	bool ret = false;
683	882
684	883	lockdep_assert_held(&kbdev->hwaccess_lock);
685		- dev_dbg(kbdev->dev, "Add pullable tail kctx %pK (s:%d)\n",
686		- (void *)kctx, js);
	884	+ dev_dbg(kbdev->dev, "Add pullable tail kctx %pK (s:%u)\n", (void *)kctx, js);
687	885
688	886	if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
689	887	list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
..	..	@@ -694,7 +892,7 @@
694	892	if (!kctx->slots_pullable) {
695	893	kbdev->js_data.nr_contexts_pullable++;
696	894	ret = true;
697		- if (!atomic_read(&kctx->atoms_pulled)) {
	895	+ if (!kbase_jsctx_atoms_pulled(kctx)) {
698	896	WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
699	897	kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF);
700	898	atomic_inc(&kbdev->js_data.nr_contexts_runnable);
..	..	@@ -718,14 +916,13 @@
718	916	*
719	917	* Return: true if caller should call kbase_backend_ctx_count_changed()
720	918	*/
721		-static bool kbase_js_ctx_list_add_pullable_head_nolock(
722		- struct kbase_device kbdev, struct kbase_context kctx, int js)
	919	+static bool kbase_js_ctx_list_add_pullable_head_nolock(struct kbase_device *kbdev,
	920	+ struct kbase_context *kctx, unsigned int js)
723	921	{
724	922	bool ret = false;
725	923
726	924	lockdep_assert_held(&kbdev->hwaccess_lock);
727		- dev_dbg(kbdev->dev, "Add pullable head kctx %pK (s:%d)\n",
728		- (void *)kctx, js);
	925	+ dev_dbg(kbdev->dev, "Add pullable head kctx %pK (s:%u)\n", (void *)kctx, js);
729	926
730	927	if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]))
731	928	list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
..	..	@@ -736,7 +933,7 @@
736	933	if (!kctx->slots_pullable) {
737	934	kbdev->js_data.nr_contexts_pullable++;
738	935	ret = true;
739		- if (!atomic_read(&kctx->atoms_pulled)) {
	936	+ if (!kbase_jsctx_atoms_pulled(kctx)) {
740	937	WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
741	938	kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF);
742	939	atomic_inc(&kbdev->js_data.nr_contexts_runnable);
..	..	@@ -763,8 +960,7 @@
763	960	* Return: true if caller should call kbase_backend_ctx_count_changed()
764	961	*/
765	962	static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev,
766		- struct kbase_context *kctx,
767		- int js)
	963	+ struct kbase_context *kctx, unsigned int js)
768	964	{
769	965	bool ret;
770	966	unsigned long flags;
..	..	@@ -794,14 +990,12 @@
794	990	* Return: true if caller should call kbase_backend_ctx_count_changed()
795	991	*/
796	992	static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev,
797		- struct kbase_context *kctx,
798		- int js)
	993	+ struct kbase_context *kctx, unsigned int js)
799	994	{
800	995	bool ret = false;
801	996
802	997	lockdep_assert_held(&kbdev->hwaccess_lock);
803		- dev_dbg(kbdev->dev, "Add unpullable tail kctx %pK (s:%d)\n",
804		- (void *)kctx, js);
	998	+ dev_dbg(kbdev->dev, "Add unpullable tail kctx %pK (s:%u)\n", (void *)kctx, js);
805	999
806	1000	list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js],
807	1001	&kbdev->js_data.ctx_list_unpullable[js][kctx->priority]);
..	..	@@ -809,7 +1003,7 @@
809	1003	if (kctx->slots_pullable == (1 << js)) {
810	1004	kbdev->js_data.nr_contexts_pullable--;
811	1005	ret = true;
812		- if (!atomic_read(&kctx->atoms_pulled)) {
	1006	+ if (!kbase_jsctx_atoms_pulled(kctx)) {
813	1007	WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
814	1008	kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF);
815	1009	atomic_dec(&kbdev->js_data.nr_contexts_runnable);
..	..	@@ -836,9 +1030,8 @@
836	1030	*
837	1031	* Return: true if caller should call kbase_backend_ctx_count_changed()
838	1032	*/
839		-static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev,
840		- struct kbase_context *kctx,
841		- int js)
	1033	+static bool kbase_js_ctx_list_remove_nolock(struct kbase_device kbdev, struct kbase_context kctx,
	1034	+ unsigned int js)
842	1035	{
843	1036	bool ret = false;
844	1037
..	..	@@ -851,7 +1044,7 @@
851	1044	if (kctx->slots_pullable == (1 << js)) {
852	1045	kbdev->js_data.nr_contexts_pullable--;
853	1046	ret = true;
854		- if (!atomic_read(&kctx->atoms_pulled)) {
	1047	+ if (!kbase_jsctx_atoms_pulled(kctx)) {
855	1048	WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
856	1049	kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF);
857	1050	atomic_dec(&kbdev->js_data.nr_contexts_runnable);
..	..	@@ -874,9 +1067,8 @@
874	1067	* Return: Context to use for specified slot.
875	1068	* NULL if no contexts present for specified slot
876	1069	*/
877		-static struct kbase_context *kbase_js_ctx_list_pop_head_nolock(
878		- struct kbase_device *kbdev,
879		- int js)
	1070	+static struct kbase_context kbase_js_ctx_list_pop_head_nolock(struct kbase_device kbdev,
	1071	+ unsigned int js)
880	1072	{
881	1073	struct kbase_context *kctx;
882	1074	int i;
..	..	@@ -892,9 +1084,8 @@
892	1084	jctx.sched_info.ctx.ctx_list_entry[js]);
893	1085
894	1086	list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]);
895		- dev_dbg(kbdev->dev,
896		- "Popped %pK from the pullable queue (s:%d)\n",
897		- (void *)kctx, js);
	1087	+ dev_dbg(kbdev->dev, "Popped %pK from the pullable queue (s:%u)\n", (void *)kctx,
	1088	+ js);
898	1089	return kctx;
899	1090	}
900	1091	return NULL;
..	..	@@ -909,8 +1100,7 @@
909	1100	* Return: Context to use for specified slot.
910	1101	* NULL if no contexts present for specified slot
911	1102	*/
912		-static struct kbase_context *kbase_js_ctx_list_pop_head(
913		- struct kbase_device *kbdev, int js)
	1103	+static struct kbase_context kbase_js_ctx_list_pop_head(struct kbase_device kbdev, unsigned int js)
914	1104	{
915	1105	struct kbase_context *kctx;
916	1106	unsigned long flags;
..	..	@@ -934,8 +1124,7 @@
934	1124	* Return: true if context can be pulled from on specified slot
935	1125	* false otherwise
936	1126	*/
937		-static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js,
938		- bool is_scheduled)
	1127	+static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled)
939	1128	{
940	1129	struct kbasep_js_device_data *js_devdata;
941	1130	struct kbase_jd_atom *katom;
..	..	@@ -954,13 +1143,15 @@
954	1143	}
955	1144	katom = jsctx_rb_peek(kctx, js);
956	1145	if (!katom) {
957		- dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%d)\n",
958		- (void *)kctx, js);
	1146	+ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js);
959	1147	return false; /* No pullable atoms */
960	1148	}
961		- if (kctx->blocked_js[js][katom->sched_priority]) {
	1149	+ if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) {
	1150	+ KBASE_KTRACE_ADD_JM_SLOT_INFO(
	1151	+ kctx->kbdev, JS_SLOT_PRIO_IS_BLOCKED, kctx, katom,
	1152	+ katom->jc, js, (unsigned int)katom->sched_priority);
962	1153	dev_dbg(kbdev->dev,
963		- "JS: kctx %pK is blocked from submitting atoms at priority %d (s:%d)\n",
	1154	+ "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n",
964	1155	(void *)kctx, katom->sched_priority, js);
965	1156	return false;
966	1157	}
..	..	@@ -981,14 +1172,14 @@
981	1172	if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
982	1173	kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) {
983	1174	dev_dbg(kbdev->dev,
984		- "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%d)\n",
	1175	+ "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n",
985	1176	(void *)katom, js);
986	1177	return false;
987	1178	}
988	1179	}
989	1180
990		- dev_dbg(kbdev->dev, "JS: Atom %pK is pullable in kctx %pK (s:%d)\n",
991		- (void )katom, (void )kctx, js);
	1181	+ dev_dbg(kbdev->dev, "JS: Atom %pK is pullable in kctx %pK (s:%u)\n", (void *)katom,
	1182	+ (void *)kctx, js);
992	1183
993	1184	return true;
994	1185	}
..	..	@@ -999,7 +1190,7 @@
999	1190	struct kbase_device *kbdev = kctx->kbdev;
1000	1191	bool ret = true;
1001	1192	bool has_dep = false, has_x_dep = false;
1002		- int js = kbase_js_get_slot(kbdev, katom);
	1193	+ unsigned int js = kbase_js_get_slot(kbdev, katom);
1003	1194	int prio = katom->sched_priority;
1004	1195	int i;
1005	1196
..	..	@@ -1007,7 +1198,7 @@
1007	1198	struct kbase_jd_atom *dep_atom = katom->dep[i].atom;
1008	1199
1009	1200	if (dep_atom) {
1010		- int dep_js = kbase_js_get_slot(kbdev, dep_atom);
	1201	+ unsigned int dep_js = kbase_js_get_slot(kbdev, dep_atom);
1011	1202	int dep_prio = dep_atom->sched_priority;
1012	1203
1013	1204	dev_dbg(kbdev->dev,
..	..	@@ -1162,7 +1353,7 @@
1162	1353	void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority)
1163	1354	{
1164	1355	struct kbase_device *kbdev = kctx->kbdev;
1165		- int js;
	1356	+ unsigned int js;
1166	1357
1167	1358	lockdep_assert_held(&kbdev->hwaccess_lock);
1168	1359
..	..	@@ -1593,6 +1784,7 @@
1593	1784	bool runpool_ctx_attr_change = false;
1594	1785	int kctx_as_nr;
1595	1786	int new_ref_count;
	1787	+ CSTD_UNUSED(kctx_as_nr);
1596	1788
1597	1789	KBASE_DEBUG_ASSERT(kbdev != NULL);
1598	1790	KBASE_DEBUG_ASSERT(kctx != NULL);
..	..	@@ -1867,9 +2059,8 @@
1867	2059	kbase_backend_timeouts_changed(kbdev);
1868	2060	}
1869	2061
1870		-static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev,
1871		- struct kbase_context *kctx,
1872		- int js)
	2062	+static bool kbasep_js_schedule_ctx(struct kbase_device kbdev, struct kbase_context kctx,
	2063	+ unsigned int js)
1873	2064	{
1874	2065	struct kbasep_js_device_data *js_devdata;
1875	2066	struct kbasep_js_kctx_info *js_kctx_info;
..	..	@@ -1877,7 +2068,7 @@
1877	2068	bool kctx_suspended = false;
1878	2069	int as_nr;
1879	2070
1880		- dev_dbg(kbdev->dev, "Scheduling kctx %pK (s:%d)\n", kctx, js);
	2071	+ dev_dbg(kbdev->dev, "Scheduling kctx %pK (s:%u)\n", kctx, js);
1881	2072
1882	2073	js_devdata = &kbdev->js_data;
1883	2074	js_kctx_info = &kctx->jctx.sched_info;
..	..	@@ -1904,8 +2095,8 @@
1904	2095	WARN_ON(as_nr == KBASEP_AS_NR_INVALID);
1905	2096	}
1906	2097	}
1907		- if (as_nr == KBASEP_AS_NR_INVALID)
1908		- return false; /* No address spaces currently available */
	2098	+ if ((as_nr < 0) \|\| (as_nr >= BASE_MAX_NR_AS))
	2099	+ return false; /* No address space currently available */
1909	2100
1910	2101	/*
1911	2102	* Atomic transaction on the Context and Run Pool begins
..	..	@@ -1976,6 +2167,7 @@
1976	2167	#endif
1977	2168	/* Cause it to leave at some later point */
1978	2169	bool retained;
	2170	+ CSTD_UNUSED(retained);
1979	2171
1980	2172	retained = kbase_ctx_sched_inc_refcount_nolock(kctx);
1981	2173	KBASE_DEBUG_ASSERT(retained);
..	..	@@ -2011,9 +2203,8 @@
2011	2203	return true;
2012	2204	}
2013	2205
2014		-static bool kbase_js_use_ctx(struct kbase_device *kbdev,
2015		- struct kbase_context *kctx,
2016		- int js)
	2206	+static bool kbase_js_use_ctx(struct kbase_device kbdev, struct kbase_context kctx,
	2207	+ unsigned int js)
2017	2208	{
2018	2209	unsigned long flags;
2019	2210
..	..	@@ -2021,9 +2212,7 @@
2021	2212
2022	2213	if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) &&
2023	2214	kbase_backend_use_ctx_sched(kbdev, kctx, js)) {
2024		-
2025		- dev_dbg(kbdev->dev,
2026		- "kctx %pK already has ASID - mark as active (s:%d)\n",
	2215	+ dev_dbg(kbdev->dev, "kctx %pK already has ASID - mark as active (s:%u)\n",
2027	2216	(void *)kctx, js);
2028	2217
2029	2218	if (kbdev->hwaccess.active_kctx[js] != kctx) {
..	..	@@ -2290,8 +2479,7 @@
2290	2479	return true;
2291	2480	}
2292	2481
2293		-static int kbase_js_get_slot(struct kbase_device *kbdev,
2294		- struct kbase_jd_atom *katom)
	2482	+static unsigned int kbase_js_get_slot(struct kbase_device kbdev, struct kbase_jd_atom katom)
2295	2483	{
2296	2484	if (katom->core_req & BASE_JD_REQ_JOB_SLOT)
2297	2485	return katom->jobslot;
..	..	@@ -2321,21 +2509,19 @@
2321	2509	/* If slot will transition from unpullable to pullable then add to
2322	2510	* pullable list
2323	2511	*/
2324		- if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) {
	2512	+ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr))
2325	2513	enqueue_required = true;
2326		- } else {
	2514	+ else
2327	2515	enqueue_required = false;
2328		- }
2329	2516
2330	2517	if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) \|\|
2331	2518	(katom->pre_dep && (katom->pre_dep->atom_flags &
2332	2519	KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) {
2333	2520	int prio = katom->sched_priority;
2334		- int js = katom->slot_nr;
	2521	+ unsigned int js = katom->slot_nr;
2335	2522	struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js];
2336	2523
2337		- dev_dbg(kctx->kbdev->dev, "Add atom %pK to X_DEP list (s:%d)\n",
2338		- (void *)katom, js);
	2524	+ dev_dbg(kctx->kbdev->dev, "Add atom %pK to X_DEP list (s:%u)\n", (void *)katom, js);
2339	2525
2340	2526	list_add_tail(&katom->queue, &queue->x_dep_head);
2341	2527	katom->atom_flags \|= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST;
..	..	@@ -2426,8 +2612,8 @@
2426	2612	*
2427	2613	* Context: Caller must hold the HW access lock
2428	2614	*/
2429		-static void kbase_js_evict_deps(struct kbase_context *kctx,
2430		- struct kbase_jd_atom *katom, int js, int prio)
	2615	+static void kbase_js_evict_deps(struct kbase_context kctx, struct kbase_jd_atom katom,
	2616	+ unsigned int js, int prio)
2431	2617	{
2432	2618	struct kbase_jd_atom *x_dep = katom->x_post_dep;
2433	2619	struct kbase_jd_atom *next_katom = katom->post_dep;
..	..	@@ -2451,15 +2637,15 @@
2451	2637	(void *)x_dep);
2452	2638
2453	2639	/* Fail if it had a data dependency. */
2454		- if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) {
	2640	+ if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)
2455	2641	x_dep->will_fail_event_code = katom->event_code;
2456		- }
	2642	+
2457	2643	if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)
2458	2644	kbase_js_move_to_tree(x_dep);
2459	2645	}
2460	2646	}
2461	2647
2462		-struct kbase_jd_atom kbase_js_pull(struct kbase_context kctx, int js)
	2648	+struct kbase_jd_atom kbase_js_pull(struct kbase_context kctx, unsigned int js)
2463	2649	{
2464	2650	struct kbase_jd_atom *katom;
2465	2651	struct kbasep_js_device_data *js_devdata;
..	..	@@ -2469,8 +2655,7 @@
2469	2655	KBASE_DEBUG_ASSERT(kctx);
2470	2656
2471	2657	kbdev = kctx->kbdev;
2472		- dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %pK (s:%d)\n",
2473		- (void *)kctx, js);
	2658	+ dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %pK (s:%u)\n", (void *)kctx, js);
2474	2659
2475	2660	js_devdata = &kbdev->js_data;
2476	2661	lockdep_assert_held(&kbdev->hwaccess_lock);
..	..	@@ -2489,13 +2674,12 @@
2489	2674
2490	2675	katom = jsctx_rb_peek(kctx, js);
2491	2676	if (!katom) {
2492		- dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%d)\n",
2493		- (void *)kctx, js);
	2677	+ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js);
2494	2678	return NULL;
2495	2679	}
2496		- if (kctx->blocked_js[js][katom->sched_priority]) {
	2680	+ if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) {
2497	2681	dev_dbg(kbdev->dev,
2498		- "JS: kctx %pK is blocked from submitting atoms at priority %d (s:%d)\n",
	2682	+ "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n",
2499	2683	(void *)kctx, katom->sched_priority, js);
2500	2684	return NULL;
2501	2685	}
..	..	@@ -2509,7 +2693,7 @@
2509	2693	* not allow multiple runs of fail-dep atoms from the same context to be
2510	2694	* present on the same slot
2511	2695	*/
2512		- if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) {
	2696	+ if (katom->pre_dep && kbase_jsctx_slot_atoms_pulled(kctx, js)) {
2513	2697	struct kbase_jd_atom *prev_atom =
2514	2698	kbase_backend_inspect_tail(kbdev, js);
2515	2699
..	..	@@ -2529,32 +2713,30 @@
2529	2713	if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) &&
2530	2714	kbase_backend_nr_atoms_on_slot(kbdev, js)) {
2531	2715	dev_dbg(kbdev->dev,
2532		- "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%d)\n",
	2716	+ "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n",
2533	2717	(void *)katom, js);
2534	2718	return NULL;
2535	2719	}
2536	2720	}
2537	2721
	2722	+ KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JS_PULL_JOB, kctx, katom,
	2723	+ katom->jc, js, katom->sched_priority);
2538	2724	kbase_ctx_flag_set(kctx, KCTX_PULLED);
2539	2725	kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js));
2540	2726
2541		- pulled = atomic_inc_return(&kctx->atoms_pulled);
	2727	+ pulled = kbase_jsctx_slot_atom_pulled_inc(kctx, katom);
2542	2728	if (pulled == 1 && !kctx->slots_pullable) {
2543	2729	WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
2544	2730	kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF);
2545	2731	atomic_inc(&kbdev->js_data.nr_contexts_runnable);
2546	2732	}
2547		- atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]);
2548		- kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++;
2549	2733	jsctx_rb_pull(kctx, katom);
2550	2734
2551	2735	kbase_ctx_sched_retain_ctx_refcount(kctx);
2552	2736
2553		- katom->atom_flags \|= KBASE_KATOM_FLAG_HOLDING_CTX_REF;
2554		-
2555	2737	katom->ticks = 0;
2556	2738
2557		- dev_dbg(kbdev->dev, "JS: successfully pulled atom %pK from kctx %pK (s:%d)\n",
	2739	+ dev_dbg(kbdev->dev, "JS: successfully pulled atom %pK from kctx %pK (s:%u)\n",
2558	2740	(void )katom, (void )kctx, js);
2559	2741
2560	2742	return katom;
..	..	@@ -2773,14 +2955,17 @@
2773	2955	struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
2774	2956	struct kbasep_js_atom_retained_state retained_state;
2775	2957	int js = katom->slot_nr;
2776		- int prio = katom->sched_priority;
	2958	+ bool slot_became_unblocked;
2777	2959	bool timer_sync = false;
2778	2960	bool context_idle = false;
2779	2961	unsigned long flags;
2780	2962	base_jd_core_req core_req = katom->core_req;
	2963	+ u64 cache_jc = katom->jc;
2781	2964
2782	2965	dev_dbg(kbdev->dev, "%s for atom %pK with event code 0x%x\n",
2783	2966	__func__, (void *)katom, katom->event_code);
	2967	+
	2968	+ KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER, kctx, katom, katom->jc, 0);
2784	2969
2785	2970	if (katom->event_code != BASE_JD_EVENT_END_RP_DONE)
2786	2971	KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom);
..	..	@@ -2792,37 +2977,27 @@
2792	2977	mutex_lock(&js_devdata->queue_mutex);
2793	2978	mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
2794	2979
2795		- atomic_dec(&kctx->atoms_pulled);
2796		- atomic_dec(&kctx->atoms_pulled_slot[js]);
2797		-
2798	2980	if (katom->event_code != BASE_JD_EVENT_END_RP_DONE)
2799	2981	atomic_dec(&katom->blocked);
2800	2982
2801	2983	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
2802	2984
2803		- kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--;
	2985	+ slot_became_unblocked = kbase_jsctx_slot_atom_pulled_dec(kctx, katom);
2804	2986
2805		- if (!atomic_read(&kctx->atoms_pulled_slot[js]) &&
2806		- jsctx_rb_none_to_pull(kctx, js))
	2987	+ if (!kbase_jsctx_slot_atoms_pulled(kctx, js) &&
	2988	+ jsctx_rb_none_to_pull(kctx, js))
2807	2989	timer_sync \|= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js);
2808	2990
2809		- /* If this slot has been blocked due to soft-stopped atoms, and all
2810		- * atoms have now been processed, then unblock the slot
	2991	+ /* If the context is now unblocked on this slot after soft-stopped
	2992	+ * atoms, then only mark it as pullable on this slot if it is not
	2993	+ * idle
2811	2994	*/
2812		- if (!kctx->atoms_pulled_slot_pri[js][prio] &&
2813		- kctx->blocked_js[js][prio]) {
2814		- kctx->blocked_js[js][prio] = false;
	2995	+ if (slot_became_unblocked && kbase_jsctx_atoms_pulled(kctx) &&
	2996	+ kbase_js_ctx_pullable(kctx, js, true))
	2997	+ timer_sync \|=
	2998	+ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js);
2815	2999
2816		- /* Only mark the slot as pullable if the context is not idle -
2817		- * that case is handled below
2818		- */
2819		- if (atomic_read(&kctx->atoms_pulled) &&
2820		- kbase_js_ctx_pullable(kctx, js, true))
2821		- timer_sync \|= kbase_js_ctx_list_add_pullable_nolock(
2822		- kbdev, kctx, js);
2823		- }
2824		-
2825		- if (!atomic_read(&kctx->atoms_pulled)) {
	3000	+ if (!kbase_jsctx_atoms_pulled(kctx)) {
2826	3001	dev_dbg(kbdev->dev,
2827	3002	"No atoms currently pulled from context %pK\n",
2828	3003	(void *)kctx);
..	..	@@ -2890,7 +3065,6 @@
2890	3065	mutex_unlock(&kctx->jctx.lock);
2891	3066	}
2892	3067
2893		- katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
2894	3068	dev_dbg(kbdev->dev, "JS: retained state %s finished",
2895	3069	kbasep_js_has_atom_finished(&retained_state) ?
2896	3070	"has" : "hasn't");
..	..	@@ -2903,6 +3077,9 @@
2903	3077	kbase_js_sched_all(kbdev);
2904	3078
2905	3079	kbase_backend_complete_wq_post_sched(kbdev, core_req);
	3080	+
	3081	+ KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER_END, kctx, NULL, cache_jc,
	3082	+ 0);
2906	3083
2907	3084	dev_dbg(kbdev->dev, "Leaving %s for atom %pK\n",
2908	3085	__func__, (void *)katom);
..	..	@@ -3113,15 +3290,16 @@
3113	3290	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
3114	3291
3115	3292	if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) {
	3293	+ bool slot_became_unblocked;
	3294	+
3116	3295	dev_dbg(kbdev->dev, "Atom %pK is in runnable_tree\n",
3117	3296	(void *)katom);
3118	3297
3119		- context_idle = !atomic_dec_return(&kctx->atoms_pulled);
3120		- atomic_dec(&kctx->atoms_pulled_slot[atom_slot]);
3121		- kctx->atoms_pulled_slot_pri[atom_slot][prio]--;
	3298	+ slot_became_unblocked =
	3299	+ kbase_jsctx_slot_atom_pulled_dec(kctx, katom);
	3300	+ context_idle = !kbase_jsctx_atoms_pulled(kctx);
3122	3301
3123		- if (!atomic_read(&kctx->atoms_pulled) &&
3124		- !kctx->slots_pullable) {
	3302	+ if (!kbase_jsctx_atoms_pulled(kctx) && !kctx->slots_pullable) {
3125	3303	WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF));
3126	3304	kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF);
3127	3305	atomic_dec(&kbdev->js_data.nr_contexts_runnable);
..	..	@@ -3129,15 +3307,14 @@
3129	3307	}
3130	3308
3131	3309	/* If this slot has been blocked due to soft-stopped atoms, and
3132		- * all atoms have now been processed, then unblock the slot
	3310	+ * all atoms have now been processed at this priority level and
	3311	+ * higher, then unblock the slot
3133	3312	*/
3134		- if (!kctx->atoms_pulled_slot_pri[atom_slot][prio]
3135		- && kctx->blocked_js[atom_slot][prio]) {
	3313	+ if (slot_became_unblocked) {
3136	3314	dev_dbg(kbdev->dev,
3137		- "kctx %pK is no longer blocked from submitting on slot %d at priority %d\n",
	3315	+ "kctx %pK is no longer blocked from submitting on slot %d at priority %d or higher\n",
3138	3316	(void *)kctx, atom_slot, prio);
3139	3317
3140		- kctx->blocked_js[atom_slot][prio] = false;
3141	3318	if (kbase_js_ctx_pullable(kctx, atom_slot, true))
3142	3319	timer_sync \|=
3143	3320	kbase_js_ctx_list_add_pullable_nolock(
..	..	@@ -3146,8 +3323,8 @@
3146	3323	}
3147	3324	WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE));
3148	3325
3149		- if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) &&
3150		- jsctx_rb_none_to_pull(kctx, atom_slot)) {
	3326	+ if (!kbase_jsctx_slot_atoms_pulled(kctx, atom_slot) &&
	3327	+ jsctx_rb_none_to_pull(kctx, atom_slot)) {
3151	3328	if (!list_empty(
3152	3329	&kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot]))
3153	3330	timer_sync \|= kbase_js_ctx_list_remove_nolock(
..	..	@@ -3160,9 +3337,9 @@
3160	3337	* re-enable submission so that context can be scheduled again.
3161	3338	*/
3162	3339	if (!kbasep_js_is_submit_allowed(js_devdata, kctx) &&
3163		- !atomic_read(&kctx->atoms_pulled) &&
3164		- !kbase_ctx_flag(kctx, KCTX_DYING)) {
3165		- int js;
	3340	+ !kbase_jsctx_atoms_pulled(kctx) &&
	3341	+ !kbase_ctx_flag(kctx, KCTX_DYING)) {
	3342	+ unsigned int js;
3166	3343
3167	3344	kbasep_js_set_submit_allowed(js_devdata, kctx);
3168	3345
..	..	@@ -3174,7 +3351,7 @@
3174	3351	}
3175	3352	} else if (katom->x_post_dep &&
3176	3353	kbasep_js_is_submit_allowed(js_devdata, kctx)) {
3177		- int js;
	3354	+ unsigned int js;
3178	3355
3179	3356	for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
3180	3357	if (kbase_js_ctx_pullable(kctx, js, true))
..	..	@@ -3285,6 +3462,11 @@
3285	3462
3286	3463	katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED;
3287	3464	dev_dbg(kbdev->dev, "Atom %pK status to HW completed\n", (void *)katom);
	3465	+ if (kbase_is_quick_reset_enabled(kbdev)) {
	3466	+ kbdev->num_of_atoms_hw_completed++;
	3467	+ if (kbdev->num_of_atoms_hw_completed >= 20)
	3468	+ kbase_disable_quick_reset(kbdev);
	3469	+ }
3288	3470
3289	3471	if (katom->event_code != BASE_JD_EVENT_DONE) {
3290	3472	kbase_js_evict_deps(kctx, katom, katom->slot_nr,
..	..	@@ -3297,7 +3479,9 @@
3297	3479	trace_sysgraph_gpu(SGR_COMPLETE, kctx->id,
3298	3480	kbase_jd_atom_id(katom->kctx, katom), katom->slot_nr);
3299	3481
	3482	+ KBASE_TLSTREAM_TL_JD_DONE_START(kbdev, katom);
3300	3483	kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0);
	3484	+ KBASE_TLSTREAM_TL_JD_DONE_END(kbdev, katom);
3301	3485
3302	3486	/* Unblock cross dependency if present */
3303	3487	if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE \|\|
..	..	@@ -3397,13 +3581,15 @@
3397	3581	return false;
3398	3582	}
3399	3583
3400		-void kbase_js_sched(struct kbase_device *kbdev, int js_mask)
	3584	+void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask)
3401	3585	{
3402	3586	struct kbasep_js_device_data *js_devdata;
3403	3587	struct kbase_context *last_active[BASE_JM_MAX_NR_SLOTS];
3404	3588	bool timer_sync = false;
3405	3589	bool ctx_waiting[BASE_JM_MAX_NR_SLOTS];
3406		- int js;
	3590	+ unsigned int js;
	3591	+
	3592	+ KBASE_TLSTREAM_TL_JS_SCHED_START(kbdev, 0);
3407	3593
3408	3594	dev_dbg(kbdev->dev, "%s kbdev %pK mask 0x%x\n",
3409	3595	__func__, (void *)kbdev, (unsigned int)js_mask);
..	..	@@ -3430,24 +3616,20 @@
3430	3616
3431	3617	if (!kctx) {
3432	3618	js_mask &= ~(1 << js);
3433		- dev_dbg(kbdev->dev,
3434		- "No kctx on pullable list (s:%d)\n",
3435		- js);
	3619	+ dev_dbg(kbdev->dev, "No kctx on pullable list (s:%u)\n", js);
3436	3620	break;
3437	3621	}
3438	3622
3439	3623	if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) {
3440	3624	context_idle = true;
3441	3625
3442		- dev_dbg(kbdev->dev,
3443		- "kctx %pK is not active (s:%d)\n",
3444		- (void *)kctx, js);
	3626	+ dev_dbg(kbdev->dev, "kctx %pK is not active (s:%u)\n", (void *)kctx,
	3627	+ js);
3445	3628
3446	3629	if (kbase_pm_context_active_handle_suspend(
3447	3630	kbdev,
3448	3631	KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) {
3449		- dev_dbg(kbdev->dev,
3450		- "Suspend pending (s:%d)\n", js);
	3632	+ dev_dbg(kbdev->dev, "Suspend pending (s:%u)\n", js);
3451	3633	/* Suspend pending - return context to
3452	3634	* queue and stop scheduling
3453	3635	*/
..	..	@@ -3460,6 +3642,8 @@
3460	3642	&kctx->jctx.sched_info.ctx.jsctx_mutex);
3461	3643	mutex_unlock(&js_devdata->queue_mutex);
3462	3644	up(&js_devdata->schedule_sem);
	3645	+ KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev,
	3646	+ 0);
3463	3647	return;
3464	3648	}
3465	3649	kbase_ctx_flag_set(kctx, KCTX_ACTIVE);
..	..	@@ -3503,16 +3687,13 @@
3503	3687	kbase_ctx_flag_clear(kctx, KCTX_PULLED);
3504	3688
3505	3689	if (!kbase_jm_kick(kbdev, 1 << js)) {
3506		- dev_dbg(kbdev->dev,
3507		- "No more jobs can be submitted (s:%d)\n",
3508		- js);
	3690	+ dev_dbg(kbdev->dev, "No more jobs can be submitted (s:%u)\n", js);
3509	3691	js_mask &= ~(1 << js);
3510	3692	}
3511	3693	if (!kbase_ctx_flag(kctx, KCTX_PULLED)) {
3512	3694	bool pullable;
3513	3695
3514		- dev_dbg(kbdev->dev,
3515		- "No atoms pulled from kctx %pK (s:%d)\n",
	3696	+ dev_dbg(kbdev->dev, "No atoms pulled from kctx %pK (s:%u)\n",
3516	3697	(void *)kctx, js);
3517	3698
3518	3699	pullable = kbase_js_ctx_pullable(kctx, js,
..	..	@@ -3596,14 +3777,15 @@
3596	3777	for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) {
3597	3778	if (kbdev->hwaccess.active_kctx[js] == last_active[js] &&
3598	3779	ctx_waiting[js]) {
3599		- dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n",
3600		- (void *)last_active[js], js);
	3780	+ dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%u)\n",
	3781	+ (void *)last_active[js], js);
3601	3782	kbdev->hwaccess.active_kctx[js] = NULL;
3602	3783	}
3603	3784	}
3604	3785
3605	3786	mutex_unlock(&js_devdata->queue_mutex);
3606	3787	up(&js_devdata->schedule_sem);
	3788	+ KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev, 0);
3607	3789	}
3608	3790
3609	3791	void kbase_js_zap_context(struct kbase_context *kctx)
..	..	@@ -3667,7 +3849,7 @@
3667	3849	*/
3668	3850	if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) {
3669	3851	unsigned long flags;
3670		- int js;
	3852	+ unsigned int js;
3671	3853
3672	3854	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
3673	3855	for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
..	..	@@ -3707,6 +3889,7 @@
3707	3889	} else {
3708	3890	unsigned long flags;
3709	3891	bool was_retained;
	3892	+ CSTD_UNUSED(was_retained);
3710	3893
3711	3894	/* Case c: didn't evict, but it is scheduled - it's in the Run
3712	3895	* Pool
..	..	@@ -3719,7 +3902,7 @@
3719	3902
3720	3903	kbasep_js_clear_submit_allowed(js_devdata, kctx);
3721	3904
3722		- /* Retain and (later) release the context whilst it is is now
	3905	+ /* Retain and (later) release the context whilst it is now
3723	3906	* disallowed from submitting jobs - ensures that someone
3724	3907	* somewhere will be removing the context later on
3725	3908	*/
..	..	@@ -3790,7 +3973,7 @@
3790	3973	{
3791	3974	struct kbase_device *kbdev;
3792	3975	unsigned long flags;
3793		- u32 js;
	3976	+ unsigned int js;
3794	3977
3795	3978	kbdev = kctx->kbdev;
3796	3979
..	..	@@ -3810,13 +3993,15 @@
3810	3993	{
3811	3994	struct priority_control_manager_device *pcm_device = kbdev->pcm_dev;
3812	3995	int req_priority, out_priority;
3813		- base_jd_prio out_jd_priority = priority;
3814	3996
3815		- if (pcm_device) {
3816		- req_priority = kbasep_js_atom_prio_to_sched_prio(priority);
3817		- out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, req_priority);
3818		- out_jd_priority = kbasep_js_sched_prio_to_atom_prio(out_priority);
3819		- }
3820		- return out_jd_priority;
	3997	+ req_priority = kbasep_js_atom_prio_to_sched_prio(priority);
	3998	+ out_priority = req_priority;
	3999	+ /* Does not use pcm defined priority check if PCM not defined or if
	4000	+ * kbasep_js_atom_prio_to_sched_prio returns an error
	4001	+ * (KBASE_JS_ATOM_SCHED_PRIO_INVALID).
	4002	+ */
	4003	+ if (pcm_device && (req_priority != KBASE_JS_ATOM_SCHED_PRIO_INVALID))
	4004	+ out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current,
	4005	+ req_priority);
	4006	+ return kbasep_js_sched_prio_to_atom_prio(kbdev, out_priority);
3821	4007	}
3822		-