forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h
....@@ -1,7 +1,7 @@
11 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
22 /*
33  *
4
- * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -29,6 +29,8 @@
2929 
3030 #include "mali_kbase_js_ctx_attr.h"
3131 
32
+#define JS_MAX_RUNNING_JOBS 8
33
+
3234 /**
3335  * kbasep_js_devdata_init - Initialize the Job Scheduler
3436  * @kbdev: The kbase_device to operate on
....@@ -36,6 +38,8 @@
3638  * The struct kbasep_js_device_data sub-structure of kbdev must be zero
3739  * initialized before passing to the kbasep_js_devdata_init() function. This is
3840  * to give efficient error path code.
41
+ *
42
+ * Return: 0 on success, error code otherwise.
3943  */
4044 int kbasep_js_devdata_init(struct kbase_device * const kbdev);
4145 
....@@ -86,6 +90,8 @@
8690  *
8791  * The struct kbase_context must be zero initialized before passing to the
8892  * kbase_js_init() function. This is to give efficient error path code.
93
+ *
94
+ * Return: 0 on success, error code otherwise.
8995  */
9096 int kbasep_js_kctx_init(struct kbase_context *const kctx);
9197 
....@@ -107,6 +113,52 @@
107113  * registered with this context.
108114  */
109115 void kbasep_js_kctx_term(struct kbase_context *kctx);
116
+
117
+/* kbase_jsctx_slot_prio_blocked_set - Set a context as being blocked for a job
118
+ *                                     slot at and below a given priority level
119
+ * @kctx: The kbase_context
120
+ * @js: The job slot
121
+ * @sched_prio: The priority levels that the context is blocked at for @js (all
122
+ *              priority levels at this level and below will be blocked)
123
+ *
124
+ * To preserve ordering and dependencies of atoms on soft-stopping (both within
125
+ * an between priority levels), a context must be marked as blocked for that
126
+ * atom's job slot, for all priority levels at or below the atom's priority.
127
+ *
128
+ * This must only be called due to an atom that was pulled from the context,
129
+ * otherwise there will be no way of unblocking the context when the atom is
130
+ * completed/unpulled.
131
+ *
132
+ * Atoms of higher priority might still be able to be pulled from the context
133
+ * on @js. This helps with starting a high priority atom as soon as possible.
134
+ */
135
+static inline void kbase_jsctx_slot_prio_blocked_set(struct kbase_context *kctx, unsigned int js,
136
+						     int sched_prio)
137
+{
138
+	struct kbase_jsctx_slot_tracking *slot_tracking =
139
+		&kctx->slot_tracking[js];
140
+
141
+	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
142
+	WARN(!slot_tracking->atoms_pulled_pri[sched_prio],
143
+	     "When marking slot %u as blocked for priority %d on a kctx, no atoms were pulled - the slot cannot become unblocked",
144
+	     js, sched_prio);
145
+
146
+	slot_tracking->blocked |= ((kbase_js_prio_bitmap_t)1) << sched_prio;
147
+	KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_BLOCKED, kctx,
148
+				      NULL, 0, js, (unsigned int)sched_prio);
149
+}
150
+
151
+/* kbase_jsctx_atoms_pulled - Return number of atoms pulled on a context
152
+ * @kctx: The kbase_context
153
+ *
154
+ * Having atoms pulled indicates the context is not idle.
155
+ *
156
+ * Return: the number of atoms pulled on @kctx
157
+ */
158
+static inline int kbase_jsctx_atoms_pulled(struct kbase_context *kctx)
159
+{
160
+	return atomic_read(&kctx->atoms_pulled_all_slots);
161
+}
110162 
111163 /**
112164  * kbasep_js_add_job - Add a job chain to the Job Scheduler,
....@@ -160,7 +212,7 @@
160212  * @kbdev: The kbase_device to operate on
161213  * @kctx:  The kbase_context to operate on
162214  * @atom: Atom to remove
163
-*
215
+ *
164216  * Completely removing a job requires several calls:
165217  * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of
166218  *   the atom
....@@ -310,9 +362,10 @@
310362 		struct kbase_context *kctx);
311363 
312364 /**
313
- * kbasep_js_runpool_release_ctx_and_katom_retained_state -  Variant of
365
+ * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of
314366  * kbasep_js_runpool_release_ctx() that handles additional
315367  * actions from completing an atom.
368
+ *
316369  * @kbdev:                KBase device
317370  * @kctx:                 KBase context
318371  * @katom_retained_state: Retained state from the atom
....@@ -335,8 +388,8 @@
335388 		struct kbasep_js_atom_retained_state *katom_retained_state);
336389 
337390 /**
338
- * kbasep_js_runpool_release_ctx_nolock -
339
- * Variant of kbase_js_runpool_release_ctx() w/out locks
391
+ * kbasep_js_runpool_release_ctx_nolock - Variant of kbase_js_runpool_release_ctx()
392
+ *                                        without locks
340393  * @kbdev: KBase device
341394  * @kctx:  KBase context
342395  *
....@@ -350,6 +403,7 @@
350403 
351404 /**
352405  * kbasep_js_schedule_privileged_ctx -  Schedule in a privileged context
406
+ *
353407  * @kbdev: KBase device
354408  * @kctx:  KBase context
355409  *
....@@ -413,7 +467,7 @@
413467  * contexts from (re)entering the runpool.
414468  *
415469  * This does not handle suspending the one privileged context: the caller must
416
- * instead do this by by suspending the GPU HW Counter Instrumentation.
470
+ * instead do this by suspending the GPU HW Counter Instrumentation.
417471  *
418472  * This will eventually cause all Power Management active references held by
419473  * contexts on the runpool to be released, without running any more atoms.
....@@ -456,19 +510,6 @@
456510 		struct kbase_jd_atom *katom);
457511 
458512 /**
459
- * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer.
460
- * @kctx:  Context Pointer
461
- * @prio:  Priority (specifies the queue together with js).
462
- * @js:    Job slot (specifies the queue together with prio).
463
- *
464
- * Pushes all possible atoms from the linked list to the ringbuffer.
465
- * Number of atoms are limited to free space in the ringbuffer and
466
- * number of available atoms in the linked list.
467
- *
468
- */
469
-void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js);
470
-
471
-/**
472513  * kbase_js_pull - Pull an atom from a context in the job scheduler for
473514  *                 execution.
474515  *
....@@ -482,7 +523,7 @@
482523  * Return: a pointer to an atom, or NULL if there are no atoms for this
483524  * slot that can be currently run.
484525  */
485
-struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js);
526
+struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js);
486527 
487528 /**
488529  * kbase_js_unpull - Return an atom to the job scheduler ringbuffer.
....@@ -563,10 +604,10 @@
563604  * been used.
564605  *
565606  */
566
-void kbase_js_sched(struct kbase_device *kbdev, int js_mask);
607
+void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask);
567608 
568609 /**
569
- * kbase_jd_zap_context - Attempt to deschedule a context that is being
610
+ * kbase_js_zap_context - Attempt to deschedule a context that is being
570611  *                        destroyed
571612  * @kctx: Context pointer
572613  *
....@@ -642,6 +683,8 @@
642683  * As with any bool, never test the return value with true.
643684  *
644685  * The caller must hold hwaccess_lock.
686
+ *
687
+ * Return: true if the context is allowed to submit jobs, false otherwise.
645688  */
646689 static inline bool kbasep_js_is_submit_allowed(
647690 		struct kbasep_js_device_data *js_devdata,
....@@ -651,8 +694,10 @@
651694 	bool is_allowed;
652695 
653696 	/* Ensure context really is scheduled in */
654
-	KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
655
-	KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
697
+	if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED),
698
+		 "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx,
699
+		 kctx->as_nr, atomic_read(&kctx->flags)))
700
+		return false;
656701 
657702 	test_bit = (u16) (1u << kctx->as_nr);
658703 
....@@ -679,8 +724,10 @@
679724 	u16 set_bit;
680725 
681726 	/* Ensure context really is scheduled in */
682
-	KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
683
-	KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
727
+	if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED),
728
+		 "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx,
729
+		 kctx->as_nr, atomic_read(&kctx->flags)))
730
+		return;
684731 
685732 	set_bit = (u16) (1u << kctx->as_nr);
686733 
....@@ -709,8 +756,10 @@
709756 	u16 clear_mask;
710757 
711758 	/* Ensure context really is scheduled in */
712
-	KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
713
-	KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
759
+	if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED),
760
+		 "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx,
761
+		 kctx->as_nr, atomic_read(&kctx->flags)))
762
+		return;
714763 
715764 	clear_bit = (u16) (1u << kctx->as_nr);
716765 	clear_mask = ~clear_bit;
....@@ -722,8 +771,9 @@
722771 }
723772 
724773 /**
725
- * kbasep_js_atom_retained_state_init_invalid -
726
- * Create an initial 'invalid' atom retained state
774
+ * kbasep_js_atom_retained_state_init_invalid - Create an initial 'invalid'
775
+ *                                              atom retained state
776
+ *
727777  * @retained_state: pointer where to create and initialize the state
728778  *
729779  * Create an initial 'invalid' atom retained state, that requires no
....@@ -743,7 +793,7 @@
743793  * @retained_state: where to copy
744794  * @katom:          where to copy from
745795  *
746
- * Copy atom state that can be made available after jd_done_nolock() is called
796
+ * Copy atom state that can be made available after kbase_jd_done_nolock() is called
747797  * on that atom.
748798  */
749799 static inline void kbasep_js_atom_retained_state_copy(
....@@ -817,9 +867,6 @@
817867 	struct kbasep_js_device_data *js_devdata;
818868 	struct kbasep_js_kctx_info *js_kctx_info;
819869 
820
-	KBASE_DEBUG_ASSERT(kbdev != NULL);
821
-	KBASE_DEBUG_ASSERT(kctx != NULL);
822
-
823870 	js_devdata = &kbdev->js_data;
824871 	js_kctx_info = &kctx->jctx.sched_info;
825872 
....@@ -827,13 +874,12 @@
827874 	lockdep_assert_held(&js_devdata->runpool_mutex);
828875 
829876 	/* Track total contexts */
830
-	KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX);
877
+	WARN_ON_ONCE(js_devdata->nr_all_contexts_running >= JS_MAX_RUNNING_JOBS);
831878 	++(js_devdata->nr_all_contexts_running);
832879 
833880 	if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
834881 		/* Track contexts that can submit jobs */
835
-		KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running <
836
-									S8_MAX);
882
+		WARN_ON_ONCE(js_devdata->nr_user_contexts_running >= JS_MAX_RUNNING_JOBS);
837883 		++(js_devdata->nr_user_contexts_running);
838884 	}
839885 }
....@@ -854,9 +900,6 @@
854900 	struct kbasep_js_device_data *js_devdata;
855901 	struct kbasep_js_kctx_info *js_kctx_info;
856902 
857
-	KBASE_DEBUG_ASSERT(kbdev != NULL);
858
-	KBASE_DEBUG_ASSERT(kctx != NULL);
859
-
860903 	js_devdata = &kbdev->js_data;
861904 	js_kctx_info = &kctx->jctx.sched_info;
862905 
....@@ -865,12 +908,12 @@
865908 
866909 	/* Track total contexts */
867910 	--(js_devdata->nr_all_contexts_running);
868
-	KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0);
911
+	WARN_ON_ONCE(js_devdata->nr_all_contexts_running < 0);
869912 
870913 	if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
871914 		/* Track contexts that can submit jobs */
872915 		--(js_devdata->nr_user_contexts_running);
873
-		KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0);
916
+		WARN_ON_ONCE(js_devdata->nr_user_contexts_running < 0);
874917 	}
875918 }
876919 
....@@ -895,8 +938,8 @@
895938 kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT];
896939 
897940 /**
898
- * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio)
899
- *                                        to relative ordering
941
+ * kbasep_js_atom_prio_to_sched_prio - Convert atom priority (base_jd_prio)
942
+ *                                     to relative ordering.
900943  * @atom_prio: Priority ID to translate.
901944  *
902945  * Atom priority values for @ref base_jd_prio cannot be compared directly to
....@@ -925,16 +968,33 @@
925968 	return kbasep_js_atom_priority_to_relative[atom_prio];
926969 }
927970 
928
-static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio)
971
+/**
972
+ * kbasep_js_sched_prio_to_atom_prio - Convert relative scheduler priority
973
+ *                                     to atom priority (base_jd_prio).
974
+ *
975
+ * @kbdev:    Device pointer
976
+ * @sched_prio: Relative scheduler priority to translate.
977
+ *
978
+ * This function will convert relative scheduler priority back into base_jd_prio
979
+ * values. It takes values which priorities are monotonically increasing
980
+ * and converts them to the corresponding base_jd_prio values. If an invalid number is
981
+ * passed in (i.e. not within the expected range) an error code is returned instead.
982
+ *
983
+ * The mapping is 1:1 and the size of the valid input range is the same as the
984
+ * size of the valid output range, i.e.
985
+ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS
986
+ *
987
+ * Return: On success: a value in the inclusive range
988
+ *         0..BASE_JD_NR_PRIO_LEVELS-1. On failure: BASE_JD_PRIO_INVALID.
989
+ */
990
+static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(struct kbase_device *kbdev,
991
+							     int sched_prio)
929992 {
930
-	unsigned int prio_idx;
931
-
932
-	KBASE_DEBUG_ASSERT(sched_prio >= 0 &&
933
-			sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT);
934
-
935
-	prio_idx = (unsigned int)sched_prio;
936
-
937
-	return kbasep_js_relative_priority_to_atom[prio_idx];
993
+	if (likely(sched_prio >= 0 && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT))
994
+		return kbasep_js_relative_priority_to_atom[sched_prio];
995
+	/* Invalid priority value if reached here */
996
+	dev_warn(kbdev->dev, "Unknown JS scheduling priority %d", sched_prio);
997
+	return BASE_JD_PRIO_INVALID;
938998 }
939999 
9401000 /**
....@@ -947,7 +1007,38 @@
9471007  *
9481008  * Return: The same or lower priority than requested.
9491009  */
950
-
9511010 base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority);
9521011 
1012
+/**
1013
+ * kbase_js_atom_runs_before - determine if atoms for the same slot have an
1014
+ *                             ordering relation
1015
+ * @kbdev: kbase device
1016
+ * @katom_a: the first atom
1017
+ * @katom_b: the second atom.
1018
+ * @order_flags: combination of KBASE_ATOM_ORDERING_FLAG_<...> for the ordering
1019
+ *               relation
1020
+ *
1021
+ * This is for making consistent decisions about the ordering of atoms when we
1022
+ * need to do pre-emption on a slot, which includes stopping existing atoms
1023
+ * when a new atom is ready to run, and also which other atoms to remove from
1024
+ * the slot when the atom in JSn_HEAD is being pre-empted.
1025
+ *
1026
+ * This only handles @katom_a and @katom_b being for the same job slot, as
1027
+ * pre-emption only operates within a slot.
1028
+ *
1029
+ * Note: there is currently no use-case for this as a sorting comparison
1030
+ * functions, hence only a boolean returned instead of int -1, 0, +1 return. If
1031
+ * required in future, a modification to do so would be better than calling
1032
+ * twice with katom_a and katom_b swapped.
1033
+ *
1034
+ * Return:
1035
+ * true if @katom_a should run before @katom_b, false otherwise.
1036
+ * A false return value does not distinguish between "no ordering relation" and
1037
+ * "@katom_a should run after @katom_b".
1038
+ */
1039
+bool kbase_js_atom_runs_before(struct kbase_device *kbdev,
1040
+			       const struct kbase_jd_atom *katom_a,
1041
+			       const struct kbase_jd_atom *katom_b,
1042
+			       const kbase_atom_ordering_flag_t order_flags);
1043
+
9531044 #endif	/* _KBASE_JM_JS_H_ */