forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -33,8 +33,12 @@
3333 static DEFINE_SPINLOCK(kbase_csf_fence_lock);
3434 #endif
3535 
36
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
37
+#define FENCE_WAIT_TIMEOUT_MS 3000
38
+#endif
39
+
3640 static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue,
37
-			bool ignore_waits);
41
+			       bool drain_queue);
3842 
3943 static void kcpu_queue_process_worker(struct work_struct *data);
4044 
....@@ -45,9 +49,13 @@
4549 {
4650 	struct kbase_context *const kctx = kcpu_queue->kctx;
4751 	struct kbase_va_region *reg;
52
+	struct kbase_mem_phy_alloc *alloc;
53
+	struct page **pages;
54
+	struct tagged_addr *pa;
55
+	long i;
4856 	int ret = 0;
4957 
50
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
58
+	lockdep_assert_held(&kcpu_queue->lock);
5159 
5260 	/* Take the processes mmap lock */
5361 	down_read(kbase_mem_get_process_mmap_lock());
....@@ -72,10 +80,24 @@
7280 		 * on the physical pages tracking object. When the last
7381 		 * reference to the tracking object is dropped the pages
7482 		 * would be unpinned if they weren't unpinned before.
83
+		 *
84
+		 * Region should be CPU cached: abort if it isn't.
7585 		 */
86
+		if (WARN_ON(!(reg->flags & KBASE_REG_CPU_CACHED))) {
87
+			ret = -EINVAL;
88
+			goto out;
89
+		}
90
+
7691 		ret = kbase_jd_user_buf_pin_pages(kctx, reg);
7792 		if (ret)
7893 			goto out;
94
+
95
+		alloc = reg->gpu_alloc;
96
+		pa = kbase_get_gpu_phy_pages(reg);
97
+		pages = alloc->imported.user_buf.pages;
98
+
99
+		for (i = 0; i < alloc->nents; i++)
100
+			pa[i] = as_tagged(page_to_phys(pages[i]));
79101 	}
80102 
81103 	current_command->type = BASE_KCPU_COMMAND_TYPE_MAP_IMPORT;
....@@ -99,7 +121,7 @@
99121 	struct kbase_va_region *reg;
100122 	int ret = 0;
101123 
102
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
124
+	lockdep_assert_held(&kcpu_queue->lock);
103125 
104126 	kbase_gpu_vm_lock(kctx);
105127 
....@@ -167,13 +189,14 @@
167189 			&kctx->csf.kcpu_queues.jit_blocked_queues;
168190 	struct kbase_kcpu_command_queue *blocked_queue;
169191 
170
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
192
+	lockdep_assert_held(&queue->lock);
193
+	lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock);
171194 
172195 	list_for_each_entry(blocked_queue,
173196 			&kctx->csf.kcpu_queues.jit_blocked_queues,
174197 			jit_blocked) {
175
-		struct kbase_kcpu_command const*const jit_alloc_cmd =
176
-				&blocked_queue->commands[blocked_queue->start_offset];
198
+		struct kbase_kcpu_command const *const jit_alloc_cmd =
199
+			&blocked_queue->commands[blocked_queue->start_offset];
177200 
178201 		WARN_ON(jit_alloc_cmd->type != BASE_KCPU_COMMAND_TYPE_JIT_ALLOC);
179202 		if (cmd->enqueue_ts < jit_alloc_cmd->enqueue_ts) {
....@@ -190,6 +213,12 @@
190213  *
191214  * @queue: The queue containing this JIT allocation
192215  * @cmd:   The JIT allocation command
216
+ *
217
+ * Return:
218
+ * * 0       - allocation OK
219
+ * * -EINVAL - missing info or JIT ID still in use
220
+ * * -EAGAIN - Retry
221
+ * * -ENOMEM - no memory. unable to allocate
193222  */
194223 static int kbase_kcpu_jit_allocate_process(
195224 		struct kbase_kcpu_command_queue *queue,
....@@ -206,23 +235,26 @@
206235 	u32 i;
207236 	int ret;
208237 
209
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
210
-
211
-	if (alloc_info->blocked) {
212
-		list_del(&queue->jit_blocked);
213
-		alloc_info->blocked = false;
214
-	}
238
+	lockdep_assert_held(&queue->lock);
215239 
216240 	if (WARN_ON(!info))
217241 		return -EINVAL;
242
+
243
+	mutex_lock(&kctx->csf.kcpu_queues.jit_lock);
218244 
219245 	/* Check if all JIT IDs are not in use */
220246 	for (i = 0; i < count; i++, info++) {
221247 		/* The JIT ID is still in use so fail the allocation */
222248 		if (kctx->jit_alloc[info->id]) {
223
-			dev_warn(kctx->kbdev->dev, "JIT ID still in use\n");
224
-			return -EINVAL;
249
+			dev_dbg(kctx->kbdev->dev, "JIT ID still in use");
250
+			ret = -EINVAL;
251
+			goto fail;
225252 		}
253
+	}
254
+
255
+	if (alloc_info->blocked) {
256
+		list_del(&queue->jit_blocked);
257
+		alloc_info->blocked = false;
226258 	}
227259 
228260 	/* Now start the allocation loop */
....@@ -238,7 +270,7 @@
238270 					break;
239271 
240272 				if (jit_cmd->type == BASE_KCPU_COMMAND_TYPE_JIT_FREE) {
241
-					u8 const*const free_ids = jit_cmd->info.jit_free.ids;
273
+					u8 const *const free_ids = jit_cmd->info.jit_free.ids;
242274 
243275 					if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) {
244276 						/*
....@@ -259,7 +291,7 @@
259291 				 */
260292 				dev_warn_ratelimited(kctx->kbdev->dev, "JIT alloc command failed: %pK\n", cmd);
261293 				ret = -ENOMEM;
262
-				goto fail;
294
+				goto fail_rollback;
263295 			}
264296 
265297 			/* There are pending frees for an active allocation
....@@ -277,7 +309,8 @@
277309 				kctx->jit_alloc[info->id] = NULL;
278310 			}
279311 
280
-			return -EAGAIN;
312
+			ret = -EAGAIN;
313
+			goto fail;
281314 		}
282315 
283316 		/* Bind it to the user provided ID. */
....@@ -289,11 +322,11 @@
289322 		 * Write the address of the JIT allocation to the user provided
290323 		 * GPU allocation.
291324 		 */
292
-		ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr),
293
-				&mapping);
325
+		ptr = kbase_vmap_prot(kctx, info->gpu_alloc_addr, sizeof(*ptr),
326
+				KBASE_REG_CPU_WR, &mapping);
294327 		if (!ptr) {
295328 			ret = -ENOMEM;
296
-			goto fail;
329
+			goto fail_rollback;
297330 		}
298331 
299332 		reg = kctx->jit_alloc[info->id];
....@@ -302,9 +335,11 @@
302335 		kbase_vunmap(kctx, &mapping);
303336 	}
304337 
338
+	mutex_unlock(&kctx->csf.kcpu_queues.jit_lock);
339
+
305340 	return 0;
306341 
307
-fail:
342
+fail_rollback:
308343 	/* Roll back completely */
309344 	for (i = 0, info = alloc_info->info; i < count; i++, info++) {
310345 		/* Free the allocations that were successful.
....@@ -317,6 +352,8 @@
317352 
318353 		kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC;
319354 	}
355
+fail:
356
+	mutex_unlock(&kctx->csf.kcpu_queues.jit_lock);
320357 
321358 	return ret;
322359 }
....@@ -328,15 +365,16 @@
328365 {
329366 	struct kbase_context *const kctx = kcpu_queue->kctx;
330367 	void __user *data = u64_to_user_ptr(alloc_info->info);
331
-	struct base_jit_alloc_info *info;
368
+	struct base_jit_alloc_info *info = NULL;
332369 	u32 count = alloc_info->count;
333370 	int ret = 0;
334371 	u32 i;
335372 
336
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
373
+	lockdep_assert_held(&kcpu_queue->lock);
337374 
338
-	if (!data || count > kcpu_queue->kctx->jit_max_allocations ||
339
-			count > ARRAY_SIZE(kctx->jit_alloc)) {
375
+	if ((count == 0) || (count > ARRAY_SIZE(kctx->jit_alloc)) ||
376
+	    (count > kcpu_queue->kctx->jit_max_allocations) || (!data) ||
377
+	    !kbase_mem_allow_alloc(kctx)) {
340378 		ret = -EINVAL;
341379 		goto out;
342380 	}
....@@ -371,11 +409,13 @@
371409 	}
372410 
373411 	current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_ALLOC;
374
-	list_add_tail(&current_command->info.jit_alloc.node,
375
-			&kctx->csf.kcpu_queues.jit_cmds_head);
376412 	current_command->info.jit_alloc.info = info;
377413 	current_command->info.jit_alloc.count = count;
378414 	current_command->info.jit_alloc.blocked = false;
415
+	mutex_lock(&kctx->csf.kcpu_queues.jit_lock);
416
+	list_add_tail(&current_command->info.jit_alloc.node,
417
+			&kctx->csf.kcpu_queues.jit_cmds_head);
418
+	mutex_unlock(&kctx->csf.kcpu_queues.jit_lock);
379419 
380420 	return 0;
381421 out_free:
....@@ -394,7 +434,9 @@
394434 		struct kbase_kcpu_command_queue *queue,
395435 		struct kbase_kcpu_command *cmd)
396436 {
397
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
437
+	lockdep_assert_held(&queue->lock);
438
+
439
+	mutex_lock(&queue->kctx->csf.kcpu_queues.jit_lock);
398440 
399441 	/* Remove this command from the jit_cmds_head list */
400442 	list_del(&cmd->info.jit_alloc.node);
....@@ -408,6 +450,8 @@
408450 		cmd->info.jit_alloc.blocked = false;
409451 	}
410452 
453
+	mutex_unlock(&queue->kctx->csf.kcpu_queues.jit_lock);
454
+
411455 	kfree(cmd->info.jit_alloc.info);
412456 }
413457 
....@@ -420,18 +464,17 @@
420464 {
421465 	struct kbase_kcpu_command_queue *blocked_queue;
422466 
423
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
467
+	lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock);
424468 
425469 	/*
426470 	 * Reschedule all queues blocked by JIT_ALLOC commands.
427471 	 * NOTE: This code traverses the list of blocked queues directly. It
428472 	 * only works as long as the queued works are not executed at the same
429473 	 * time. This precondition is true since we're holding the
430
-	 * kbase_csf_kcpu_queue_context.lock .
474
+	 * kbase_csf_kcpu_queue_context.jit_lock .
431475 	 */
432
-	list_for_each_entry(blocked_queue,
433
-			&kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked)
434
-		queue_work(kctx->csf.kcpu_queues.wq, &blocked_queue->work);
476
+	list_for_each_entry(blocked_queue, &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked)
477
+		queue_work(blocked_queue->wq, &blocked_queue->work);
435478 }
436479 
437480 static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue,
....@@ -448,17 +491,18 @@
448491 	if (WARN_ON(!ids))
449492 		return -EINVAL;
450493 
451
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
494
+	lockdep_assert_held(&queue->lock);
495
+	mutex_lock(&kctx->csf.kcpu_queues.jit_lock);
452496 
453
-	KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END(
454
-		queue->kctx->kbdev, queue);
497
+	KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END(queue->kctx->kbdev,
498
+									   queue);
455499 
456500 	for (i = 0; i < count; i++) {
457501 		u64 pages_used = 0;
458502 		int item_err = 0;
459503 
460504 		if (!kctx->jit_alloc[ids[i]]) {
461
-			dev_warn(kctx->kbdev->dev, "invalid JIT free ID\n");
505
+			dev_dbg(kctx->kbdev->dev, "invalid JIT free ID");
462506 			rc = -EINVAL;
463507 			item_err = rc;
464508 		} else {
....@@ -480,15 +524,17 @@
480524 			queue->kctx->kbdev, queue, item_err, pages_used);
481525 	}
482526 
483
-	/* Free the list of ids */
484
-	kfree(ids);
485
-
486527 	/*
487528 	 * Remove this command from the jit_cmds_head list and retry pending
488529 	 * allocations.
489530 	 */
490531 	list_del(&cmd->info.jit_free.node);
491532 	kbase_kcpu_jit_retry_pending_allocs(kctx);
533
+
534
+	mutex_unlock(&kctx->csf.kcpu_queues.jit_lock);
535
+
536
+	/* Free the list of ids */
537
+	kfree(ids);
492538 
493539 	return rc;
494540 }
....@@ -505,7 +551,7 @@
505551 	int ret;
506552 	u32 i;
507553 
508
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
554
+	lockdep_assert_held(&kcpu_queue->lock);
509555 
510556 	/* Sanity checks */
511557 	if (!count || count > ARRAY_SIZE(kctx->jit_alloc)) {
....@@ -551,10 +597,12 @@
551597 	}
552598 
553599 	current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_FREE;
554
-	list_add_tail(&current_command->info.jit_free.node,
555
-			&kctx->csf.kcpu_queues.jit_cmds_head);
556600 	current_command->info.jit_free.ids = ids;
557601 	current_command->info.jit_free.count = count;
602
+	mutex_lock(&kctx->csf.kcpu_queues.jit_lock);
603
+	list_add_tail(&current_command->info.jit_free.node,
604
+			&kctx->csf.kcpu_queues.jit_cmds_head);
605
+	mutex_unlock(&kctx->csf.kcpu_queues.jit_lock);
558606 
559607 	return 0;
560608 out_free:
....@@ -563,6 +611,7 @@
563611 	return ret;
564612 }
565613 
614
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
566615 static int kbase_csf_queue_group_suspend_prepare(
567616 		struct kbase_kcpu_command_queue *kcpu_queue,
568617 		struct base_kcpu_command_group_suspend_info *suspend_buf,
....@@ -570,18 +619,19 @@
570619 {
571620 	struct kbase_context *const kctx = kcpu_queue->kctx;
572621 	struct kbase_suspend_copy_buffer *sus_buf = NULL;
622
+	const u32 csg_suspend_buf_size =
623
+		kctx->kbdev->csf.global_iface.groups[0].suspend_size;
573624 	u64 addr = suspend_buf->buffer;
574625 	u64 page_addr = addr & PAGE_MASK;
575
-	u64 end_addr = addr + suspend_buf->size - 1;
626
+	u64 end_addr = addr + csg_suspend_buf_size - 1;
576627 	u64 last_page_addr = end_addr & PAGE_MASK;
577628 	int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1;
578629 	int pinned_pages = 0, ret = 0;
579630 	struct kbase_va_region *reg;
580631 
581
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
632
+	lockdep_assert_held(&kcpu_queue->lock);
582633 
583
-	if (suspend_buf->size <
584
-			kctx->kbdev->csf.global_iface.groups[0].suspend_size)
634
+	if (suspend_buf->size < csg_suspend_buf_size)
585635 		return -EINVAL;
586636 
587637 	ret = kbase_csf_queue_group_handle_is_valid(kctx,
....@@ -593,7 +643,7 @@
593643 	if (!sus_buf)
594644 		return -ENOMEM;
595645 
596
-	sus_buf->size = suspend_buf->size;
646
+	sus_buf->size = csg_suspend_buf_size;
597647 	sus_buf->nr_pages = nr_pages;
598648 	sus_buf->offset = addr & ~PAGE_MASK;
599649 
....@@ -629,10 +679,11 @@
629679 		struct tagged_addr *page_array;
630680 		u64 start, end, i;
631681 
632
-		if (!(reg->flags & BASE_MEM_SAME_VA) ||
633
-				reg->nr_pages < nr_pages ||
634
-				kbase_reg_current_backed_size(reg) !=
635
-					reg->nr_pages) {
682
+		if (((reg->flags & KBASE_REG_ZONE_MASK) != KBASE_REG_ZONE_SAME_VA) ||
683
+		    (kbase_reg_current_backed_size(reg) < nr_pages) ||
684
+		    !(reg->flags & KBASE_REG_CPU_WR) ||
685
+		    (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) ||
686
+		    (kbase_is_region_shrinkable(reg)) || (kbase_va_region_is_no_user_free(reg))) {
636687 			ret = -EINVAL;
637688 			goto out_clean_pages;
638689 		}
....@@ -676,14 +727,14 @@
676727 {
677728 	return kbase_csf_queue_group_suspend(kctx, sus_buf, group_handle);
678729 }
730
+#endif
679731 
680732 static enum kbase_csf_event_callback_action event_cqs_callback(void *param)
681733 {
682734 	struct kbase_kcpu_command_queue *kcpu_queue =
683735 		(struct kbase_kcpu_command_queue *)param;
684
-	struct kbase_context *const kctx = kcpu_queue->kctx;
685736 
686
-	queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work);
737
+	queue_work(kcpu_queue->wq, &kcpu_queue->work);
687738 
688739 	return KBASE_CSF_EVENT_CALLBACK_KEEP;
689740 }
....@@ -713,7 +764,7 @@
713764 {
714765 	u32 i;
715766 
716
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
767
+	lockdep_assert_held(&queue->lock);
717768 
718769 	if (WARN_ON(!cqs_wait->objs))
719770 		return -EINVAL;
....@@ -727,10 +778,10 @@
727778 						cqs_wait->objs[i].addr, &mapping);
728779 
729780 			if (!queue->command_started) {
730
-				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START(
731
-					kbdev, queue);
781
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START(kbdev,
782
+											 queue);
732783 				queue->command_started = true;
733
-				KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_START,
784
+				KBASE_KTRACE_ADD_CSF_KCPU(kbdev, KCPU_CQS_WAIT_START,
734785 						   queue, cqs_wait->nr_objs, 0);
735786 			}
736787 
....@@ -741,24 +792,24 @@
741792 				return -EINVAL;
742793 			}
743794 
744
-			sig_set = evt[BASEP_EVENT_VAL_INDEX] > cqs_wait->objs[i].val;
795
+			sig_set =
796
+				evt[BASEP_EVENT32_VAL_OFFSET / sizeof(u32)] > cqs_wait->objs[i].val;
745797 			if (sig_set) {
746798 				bool error = false;
747799 
748800 				bitmap_set(cqs_wait->signaled, i, 1);
749801 				if ((cqs_wait->inherit_err_flags & (1U << i)) &&
750
-				    evt[BASEP_EVENT_ERR_INDEX] > 0) {
802
+				    evt[BASEP_EVENT32_ERR_OFFSET / sizeof(u32)] > 0) {
751803 					queue->has_error = true;
752804 					error = true;
753805 				}
754806 
755
-				KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_END,
807
+				KBASE_KTRACE_ADD_CSF_KCPU(kbdev, KCPU_CQS_WAIT_END,
756808 						queue, cqs_wait->objs[i].addr,
757809 						error);
758810 
759811 				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END(
760
-					kbdev, queue,
761
-					evt[BASEP_EVENT_ERR_INDEX]);
812
+					kbdev, queue, evt[BASEP_EVENT32_ERR_OFFSET / sizeof(u32)]);
762813 				queue->command_started = false;
763814 			}
764815 
....@@ -775,14 +826,36 @@
775826 	return bitmap_full(cqs_wait->signaled, cqs_wait->nr_objs);
776827 }
777828 
829
+static inline bool kbase_kcpu_cqs_is_data_type_valid(u8 data_type)
830
+{
831
+	return data_type == BASEP_CQS_DATA_TYPE_U32 || data_type == BASEP_CQS_DATA_TYPE_U64;
832
+}
833
+
834
+static inline bool kbase_kcpu_cqs_is_aligned(u64 addr, u8 data_type)
835
+{
836
+	BUILD_BUG_ON(BASEP_EVENT32_ALIGN_BYTES != BASEP_EVENT32_SIZE_BYTES);
837
+	BUILD_BUG_ON(BASEP_EVENT64_ALIGN_BYTES != BASEP_EVENT64_SIZE_BYTES);
838
+	WARN_ON(!kbase_kcpu_cqs_is_data_type_valid(data_type));
839
+
840
+	switch (data_type) {
841
+	default:
842
+		return false;
843
+	case BASEP_CQS_DATA_TYPE_U32:
844
+		return (addr & (BASEP_EVENT32_ALIGN_BYTES - 1)) == 0;
845
+	case BASEP_CQS_DATA_TYPE_U64:
846
+		return (addr & (BASEP_EVENT64_ALIGN_BYTES - 1)) == 0;
847
+	}
848
+}
849
+
778850 static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue,
779851 		struct base_kcpu_command_cqs_wait_info *cqs_wait_info,
780852 		struct kbase_kcpu_command *current_command)
781853 {
782854 	struct base_cqs_wait_info *objs;
783855 	unsigned int nr_objs = cqs_wait_info->nr_objs;
856
+	unsigned int i;
784857 
785
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
858
+	lockdep_assert_held(&queue->lock);
786859 
787860 	if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS)
788861 		return -EINVAL;
....@@ -798,6 +871,17 @@
798871 			nr_objs * sizeof(*objs))) {
799872 		kfree(objs);
800873 		return -ENOMEM;
874
+	}
875
+
876
+	/* Check the CQS objects as early as possible. By checking their alignment
877
+	 * (required alignment equals to size for Sync32 and Sync64 objects), we can
878
+	 * prevent overrunning the supplied event page.
879
+	 */
880
+	for (i = 0; i < nr_objs; i++) {
881
+		if (!kbase_kcpu_cqs_is_aligned(objs[i].addr, BASEP_CQS_DATA_TYPE_U32)) {
882
+			kfree(objs);
883
+			return -EINVAL;
884
+		}
801885 	}
802886 
803887 	if (++queue->cqs_wait_count == 1) {
....@@ -836,7 +920,7 @@
836920 {
837921 	unsigned int i;
838922 
839
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
923
+	lockdep_assert_held(&queue->lock);
840924 
841925 	if (WARN_ON(!cqs_set->objs))
842926 		return;
....@@ -848,22 +932,20 @@
848932 		evt = (u32 *)kbase_phy_alloc_mapping_get(
849933 			queue->kctx, cqs_set->objs[i].addr, &mapping);
850934 
851
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET(kbdev, queue,
852
-								  evt ? 0 : 1);
935
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET(kbdev, queue, evt ? 0 : 1);
853936 
854937 		if (!evt) {
855938 			dev_warn(kbdev->dev,
856939 				"Sync memory %llx already freed", cqs_set->objs[i].addr);
857940 			queue->has_error = true;
858941 		} else {
859
-			evt[BASEP_EVENT_ERR_INDEX] = queue->has_error;
942
+			evt[BASEP_EVENT32_ERR_OFFSET / sizeof(u32)] = queue->has_error;
860943 			/* Set to signaled */
861
-			evt[BASEP_EVENT_VAL_INDEX]++;
944
+			evt[BASEP_EVENT32_VAL_OFFSET / sizeof(u32)]++;
862945 			kbase_phy_alloc_mapping_put(queue->kctx, mapping);
863946 
864
-			KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_SET,
865
-					queue, cqs_set->objs[i].addr,
866
-					evt[BASEP_EVENT_ERR_INDEX]);
947
+			KBASE_KTRACE_ADD_CSF_KCPU(kbdev, KCPU_CQS_SET, queue, cqs_set->objs[i].addr,
948
+						  evt[BASEP_EVENT32_ERR_OFFSET / sizeof(u32)]);
867949 		}
868950 	}
869951 
....@@ -878,11 +960,11 @@
878960 		struct base_kcpu_command_cqs_set_info *cqs_set_info,
879961 		struct kbase_kcpu_command *current_command)
880962 {
881
-	struct kbase_context *const kctx = kcpu_queue->kctx;
882963 	struct base_cqs_set *objs;
883964 	unsigned int nr_objs = cqs_set_info->nr_objs;
965
+	unsigned int i;
884966 
885
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
967
+	lockdep_assert_held(&kcpu_queue->lock);
886968 
887969 	if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS)
888970 		return -EINVAL;
....@@ -898,6 +980,17 @@
898980 			nr_objs * sizeof(*objs))) {
899981 		kfree(objs);
900982 		return -ENOMEM;
983
+	}
984
+
985
+	/* Check the CQS objects as early as possible. By checking their alignment
986
+	 * (required alignment equals to size for Sync32 and Sync64 objects), we can
987
+	 * prevent overrunning the supplied event page.
988
+	 */
989
+	for (i = 0; i < nr_objs; i++) {
990
+		if (!kbase_kcpu_cqs_is_aligned(objs[i].addr, BASEP_CQS_DATA_TYPE_U32)) {
991
+			kfree(objs);
992
+			return -EINVAL;
993
+		}
901994 	}
902995 
903996 	current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET;
....@@ -932,7 +1025,7 @@
9321025 {
9331026 	u32 i;
9341027 
935
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
1028
+	lockdep_assert_held(&queue->lock);
9361029 
9371030 	if (WARN_ON(!cqs_wait_operation->objs))
9381031 		return -EINVAL;
....@@ -942,12 +1035,16 @@
9421035 		if (!test_bit(i, cqs_wait_operation->signaled)) {
9431036 			struct kbase_vmap_struct *mapping;
9441037 			bool sig_set;
945
-			u64 *evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx,
946
-						cqs_wait_operation->objs[i].addr, &mapping);
1038
+			uintptr_t evt = (uintptr_t)kbase_phy_alloc_mapping_get(
1039
+				queue->kctx, cqs_wait_operation->objs[i].addr, &mapping);
1040
+			u64 val = 0;
9471041 
948
-			/* GPUCORE-28172 RDT to review */
949
-			if (!queue->command_started)
1042
+			if (!queue->command_started) {
9501043 				queue->command_started = true;
1044
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START(
1045
+					kbdev, queue);
1046
+			}
1047
+
9511048 
9521049 			if (!evt) {
9531050 				dev_warn(kbdev->dev,
....@@ -956,15 +1053,32 @@
9561053 				return -EINVAL;
9571054 			}
9581055 
1056
+			switch (cqs_wait_operation->objs[i].data_type) {
1057
+			default:
1058
+				WARN_ON(!kbase_kcpu_cqs_is_data_type_valid(
1059
+					cqs_wait_operation->objs[i].data_type));
1060
+				kbase_phy_alloc_mapping_put(queue->kctx, mapping);
1061
+				queue->has_error = true;
1062
+				return -EINVAL;
1063
+			case BASEP_CQS_DATA_TYPE_U32:
1064
+				val = *(u32 *)evt;
1065
+				evt += BASEP_EVENT32_ERR_OFFSET - BASEP_EVENT32_VAL_OFFSET;
1066
+				break;
1067
+			case BASEP_CQS_DATA_TYPE_U64:
1068
+				val = *(u64 *)evt;
1069
+				evt += BASEP_EVENT64_ERR_OFFSET - BASEP_EVENT64_VAL_OFFSET;
1070
+				break;
1071
+			}
1072
+
9591073 			switch (cqs_wait_operation->objs[i].operation) {
9601074 			case BASEP_CQS_WAIT_OPERATION_LE:
961
-				sig_set = *evt <= cqs_wait_operation->objs[i].val;
1075
+				sig_set = val <= cqs_wait_operation->objs[i].val;
9621076 				break;
9631077 			case BASEP_CQS_WAIT_OPERATION_GT:
964
-				sig_set = *evt > cqs_wait_operation->objs[i].val;
1078
+				sig_set = val > cqs_wait_operation->objs[i].val;
9651079 				break;
9661080 			default:
967
-				dev_warn(kbdev->dev,
1081
+				dev_dbg(kbdev->dev,
9681082 					"Unsupported CQS wait operation %d", cqs_wait_operation->objs[i].operation);
9691083 
9701084 				kbase_phy_alloc_mapping_put(queue->kctx, mapping);
....@@ -973,27 +1087,15 @@
9731087 				return -EINVAL;
9741088 			}
9751089 
976
-			/* Increment evt up to the error_state value depending on the CQS data type */
977
-			switch (cqs_wait_operation->objs[i].data_type) {
978
-			default:
979
-				dev_warn(kbdev->dev, "Unreachable data_type=%d", cqs_wait_operation->objs[i].data_type);
980
-			/* Fallthrough - hint to compiler that there's really only 2 options at present */
981
-			case BASEP_CQS_DATA_TYPE_U32:
982
-				evt = (u64 *)((u8 *)evt + sizeof(u32));
983
-				break;
984
-			case BASEP_CQS_DATA_TYPE_U64:
985
-				evt = (u64 *)((u8 *)evt + sizeof(u64));
986
-				break;
987
-			}
988
-
9891090 			if (sig_set) {
9901091 				bitmap_set(cqs_wait_operation->signaled, i, 1);
9911092 				if ((cqs_wait_operation->inherit_err_flags & (1U << i)) &&
992
-				    *evt > 0) {
1093
+				    *(u32 *)evt > 0) {
9931094 					queue->has_error = true;
9941095 				}
9951096 
996
-				/* GPUCORE-28172 RDT to review */
1097
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END(
1098
+					kbdev, queue, *(u32 *)evt);
9971099 
9981100 				queue->command_started = false;
9991101 			}
....@@ -1017,8 +1119,9 @@
10171119 {
10181120 	struct base_cqs_wait_operation_info *objs;
10191121 	unsigned int nr_objs = cqs_wait_operation_info->nr_objs;
1122
+	unsigned int i;
10201123 
1021
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
1124
+	lockdep_assert_held(&queue->lock);
10221125 
10231126 	if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS)
10241127 		return -EINVAL;
....@@ -1034,6 +1137,18 @@
10341137 			nr_objs * sizeof(*objs))) {
10351138 		kfree(objs);
10361139 		return -ENOMEM;
1140
+	}
1141
+
1142
+	/* Check the CQS objects as early as possible. By checking their alignment
1143
+	 * (required alignment equals to size for Sync32 and Sync64 objects), we can
1144
+	 * prevent overrunning the supplied event page.
1145
+	 */
1146
+	for (i = 0; i < nr_objs; i++) {
1147
+		if (!kbase_kcpu_cqs_is_data_type_valid(objs[i].data_type) ||
1148
+		    !kbase_kcpu_cqs_is_aligned(objs[i].addr, objs[i].data_type)) {
1149
+			kfree(objs);
1150
+			return -EINVAL;
1151
+		}
10371152 	}
10381153 
10391154 	if (++queue->cqs_wait_count == 1) {
....@@ -1066,6 +1181,44 @@
10661181 	return 0;
10671182 }
10681183 
1184
+static void kbasep_kcpu_cqs_do_set_operation_32(struct kbase_kcpu_command_queue *queue,
1185
+						uintptr_t evt, u8 operation, u64 val)
1186
+{
1187
+	struct kbase_device *kbdev = queue->kctx->kbdev;
1188
+
1189
+	switch (operation) {
1190
+	case BASEP_CQS_SET_OPERATION_ADD:
1191
+		*(u32 *)evt += (u32)val;
1192
+		break;
1193
+	case BASEP_CQS_SET_OPERATION_SET:
1194
+		*(u32 *)evt = val;
1195
+		break;
1196
+	default:
1197
+		dev_dbg(kbdev->dev, "Unsupported CQS set operation %d", operation);
1198
+		queue->has_error = true;
1199
+		break;
1200
+	}
1201
+}
1202
+
1203
+static void kbasep_kcpu_cqs_do_set_operation_64(struct kbase_kcpu_command_queue *queue,
1204
+						uintptr_t evt, u8 operation, u64 val)
1205
+{
1206
+	struct kbase_device *kbdev = queue->kctx->kbdev;
1207
+
1208
+	switch (operation) {
1209
+	case BASEP_CQS_SET_OPERATION_ADD:
1210
+		*(u64 *)evt += val;
1211
+		break;
1212
+	case BASEP_CQS_SET_OPERATION_SET:
1213
+		*(u64 *)evt = val;
1214
+		break;
1215
+	default:
1216
+		dev_dbg(kbdev->dev, "Unsupported CQS set operation %d", operation);
1217
+		queue->has_error = true;
1218
+		break;
1219
+	}
1220
+}
1221
+
10691222 static void kbase_kcpu_cqs_set_operation_process(
10701223 		struct kbase_device *kbdev,
10711224 		struct kbase_kcpu_command_queue *queue,
....@@ -1073,57 +1226,49 @@
10731226 {
10741227 	unsigned int i;
10751228 
1076
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
1229
+	lockdep_assert_held(&queue->lock);
10771230 
10781231 	if (WARN_ON(!cqs_set_operation->objs))
10791232 		return;
10801233 
10811234 	for (i = 0; i < cqs_set_operation->nr_objs; i++) {
10821235 		struct kbase_vmap_struct *mapping;
1083
-		u64 *evt;
1236
+		uintptr_t evt;
10841237 
1085
-		evt = (u64 *)kbase_phy_alloc_mapping_get(
1238
+		evt = (uintptr_t)kbase_phy_alloc_mapping_get(
10861239 			queue->kctx, cqs_set_operation->objs[i].addr, &mapping);
1087
-
1088
-		/* GPUCORE-28172 RDT to review */
10891240 
10901241 		if (!evt) {
10911242 			dev_warn(kbdev->dev,
10921243 				"Sync memory %llx already freed", cqs_set_operation->objs[i].addr);
10931244 			queue->has_error = true;
10941245 		} else {
1095
-			switch (cqs_set_operation->objs[i].operation) {
1096
-			case BASEP_CQS_SET_OPERATION_ADD:
1097
-				*evt += cqs_set_operation->objs[i].val;
1098
-				break;
1099
-			case BASEP_CQS_SET_OPERATION_SET:
1100
-				*evt = cqs_set_operation->objs[i].val;
1101
-				break;
1102
-			default:
1103
-				dev_warn(kbdev->dev,
1104
-					"Unsupported CQS set operation %d", cqs_set_operation->objs[i].operation);
1105
-				queue->has_error = true;
1106
-				break;
1107
-			}
1246
+			struct base_cqs_set_operation_info *obj = &cqs_set_operation->objs[i];
11081247 
1109
-			/* Increment evt up to the error_state value depending on the CQS data type */
1110
-			switch (cqs_set_operation->objs[i].data_type) {
1248
+			switch (obj->data_type) {
11111249 			default:
1112
-				dev_warn(kbdev->dev, "Unreachable data_type=%d", cqs_set_operation->objs[i].data_type);
1113
-			/* Fallthrough - hint to compiler that there's really only 2 options at present */
1250
+				WARN_ON(!kbase_kcpu_cqs_is_data_type_valid(obj->data_type));
1251
+				queue->has_error = true;
1252
+				goto skip_err_propagation;
11141253 			case BASEP_CQS_DATA_TYPE_U32:
1115
-				evt = (u64 *)((u8 *)evt + sizeof(u32));
1254
+				kbasep_kcpu_cqs_do_set_operation_32(queue, evt, obj->operation,
1255
+								    obj->val);
1256
+				evt += BASEP_EVENT32_ERR_OFFSET - BASEP_EVENT32_VAL_OFFSET;
11161257 				break;
11171258 			case BASEP_CQS_DATA_TYPE_U64:
1118
-				evt = (u64 *)((u8 *)evt + sizeof(u64));
1259
+				kbasep_kcpu_cqs_do_set_operation_64(queue, evt, obj->operation,
1260
+								    obj->val);
1261
+				evt += BASEP_EVENT64_ERR_OFFSET - BASEP_EVENT64_VAL_OFFSET;
11191262 				break;
11201263 			}
11211264 
1122
-			/* GPUCORE-28172 RDT to review */
1265
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION(
1266
+				kbdev, queue, *(u32 *)evt ? 1 : 0);
11231267 
11241268 			/* Always propagate errors */
1125
-			*evt = queue->has_error;
1269
+			*(u32 *)evt = queue->has_error;
11261270 
1271
+skip_err_propagation:
11271272 			kbase_phy_alloc_mapping_put(queue->kctx, mapping);
11281273 		}
11291274 	}
....@@ -1139,11 +1284,11 @@
11391284 		struct base_kcpu_command_cqs_set_operation_info *cqs_set_operation_info,
11401285 		struct kbase_kcpu_command *current_command)
11411286 {
1142
-	struct kbase_context *const kctx = kcpu_queue->kctx;
11431287 	struct base_cqs_set_operation_info *objs;
11441288 	unsigned int nr_objs = cqs_set_operation_info->nr_objs;
1289
+	unsigned int i;
11451290 
1146
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
1291
+	lockdep_assert_held(&kcpu_queue->lock);
11471292 
11481293 	if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS)
11491294 		return -EINVAL;
....@@ -1159,6 +1304,18 @@
11591304 			nr_objs * sizeof(*objs))) {
11601305 		kfree(objs);
11611306 		return -ENOMEM;
1307
+	}
1308
+
1309
+	/* Check the CQS objects as early as possible. By checking their alignment
1310
+	 * (required alignment equals to size for Sync32 and Sync64 objects), we can
1311
+	 * prevent overrunning the supplied event page.
1312
+	 */
1313
+	for (i = 0; i < nr_objs; i++) {
1314
+		if (!kbase_kcpu_cqs_is_data_type_valid(objs[i].data_type) ||
1315
+		    !kbase_kcpu_cqs_is_aligned(objs[i].addr, objs[i].data_type)) {
1316
+			kfree(objs);
1317
+			return -EINVAL;
1318
+		}
11621319 	}
11631320 
11641321 	current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION;
....@@ -1182,20 +1339,23 @@
11821339 	struct kbase_kcpu_command_queue *kcpu_queue = fence_info->kcpu_queue;
11831340 	struct kbase_context *const kctx = kcpu_queue->kctx;
11841341 
1185
-	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END, kcpu_queue,
1342
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
1343
+	/* Fence gets signaled. Deactivate the timer for fence-wait timeout */
1344
+	del_timer(&kcpu_queue->fence_timeout);
1345
+#endif
1346
+	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_FENCE_WAIT_END, kcpu_queue,
11861347 				  fence->context, fence->seqno);
11871348 
11881349 	/* Resume kcpu command queue processing. */
1189
-	queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work);
1350
+	queue_work(kcpu_queue->wq, &kcpu_queue->work);
11901351 }
11911352 
1192
-static void kbase_kcpu_fence_wait_cancel(
1193
-		struct kbase_kcpu_command_queue *kcpu_queue,
1194
-		struct kbase_kcpu_command_fence_info *fence_info)
1353
+static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_queue,
1354
+					  struct kbase_kcpu_command_fence_info *fence_info)
11951355 {
11961356 	struct kbase_context *const kctx = kcpu_queue->kctx;
11971357 
1198
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
1358
+	lockdep_assert_held(&kcpu_queue->lock);
11991359 
12001360 	if (WARN_ON(!fence_info->fence))
12011361 		return;
....@@ -1204,8 +1364,15 @@
12041364 		bool removed = dma_fence_remove_callback(fence_info->fence,
12051365 				&fence_info->fence_cb);
12061366 
1367
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
1368
+		/* Fence-wait cancelled or fence signaled. In the latter case
1369
+		 * the timer would already have been deactivated inside
1370
+		 * kbase_csf_fence_wait_callback().
1371
+		 */
1372
+		del_timer_sync(&kcpu_queue->fence_timeout);
1373
+#endif
12071374 		if (removed)
1208
-			KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END,
1375
+			KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_FENCE_WAIT_END,
12091376 					kcpu_queue, fence_info->fence->context,
12101377 					fence_info->fence->seqno);
12111378 	}
....@@ -1216,6 +1383,80 @@
12161383 
12171384 	fence_info->fence = NULL;
12181385 }
1386
+
1387
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
1388
+/**
1389
+ * fence_timeout_callback() - Timeout callback function for fence-wait
1390
+ *
1391
+ * @timer: Timer struct
1392
+ *
1393
+ * Context and seqno of the timed-out fence will be displayed in dmesg.
1394
+ * If the fence has been signalled a work will be enqueued to process
1395
+ * the fence-wait without displaying debugging information.
1396
+ */
1397
+static void fence_timeout_callback(struct timer_list *timer)
1398
+{
1399
+	struct kbase_kcpu_command_queue *kcpu_queue =
1400
+		container_of(timer, struct kbase_kcpu_command_queue, fence_timeout);
1401
+	struct kbase_context *const kctx = kcpu_queue->kctx;
1402
+	struct kbase_kcpu_command *cmd = &kcpu_queue->commands[kcpu_queue->start_offset];
1403
+	struct kbase_kcpu_command_fence_info *fence_info;
1404
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
1405
+	struct fence *fence;
1406
+#else
1407
+	struct dma_fence *fence;
1408
+#endif
1409
+	struct kbase_sync_fence_info info;
1410
+
1411
+	if (cmd->type != BASE_KCPU_COMMAND_TYPE_FENCE_WAIT) {
1412
+		dev_err(kctx->kbdev->dev,
1413
+			"%s: Unexpected command type %d in ctx:%d_%d kcpu queue:%u", __func__,
1414
+			cmd->type, kctx->tgid, kctx->id, kcpu_queue->id);
1415
+		return;
1416
+	}
1417
+
1418
+	fence_info = &cmd->info.fence;
1419
+
1420
+	fence = kbase_fence_get(fence_info);
1421
+	if (!fence) {
1422
+		dev_err(kctx->kbdev->dev, "no fence found in ctx:%d_%d kcpu queue:%u", kctx->tgid,
1423
+			kctx->id, kcpu_queue->id);
1424
+		return;
1425
+	}
1426
+
1427
+	kbase_sync_fence_info_get(fence, &info);
1428
+
1429
+	if (info.status == 1) {
1430
+		queue_work(kcpu_queue->wq, &kcpu_queue->work);
1431
+	} else if (info.status == 0) {
1432
+		dev_warn(kctx->kbdev->dev, "fence has not yet signalled in %ums",
1433
+			 FENCE_WAIT_TIMEOUT_MS);
1434
+		dev_warn(kctx->kbdev->dev,
1435
+			 "ctx:%d_%d kcpu queue:%u still waiting for fence[%pK] context#seqno:%s",
1436
+			 kctx->tgid, kctx->id, kcpu_queue->id, fence, info.name);
1437
+	} else {
1438
+		dev_warn(kctx->kbdev->dev, "fence has got error");
1439
+		dev_warn(kctx->kbdev->dev,
1440
+			 "ctx:%d_%d kcpu queue:%u faulty fence[%pK] context#seqno:%s error(%d)",
1441
+			 kctx->tgid, kctx->id, kcpu_queue->id, fence, info.name, info.status);
1442
+	}
1443
+
1444
+	kbase_fence_put(fence);
1445
+}
1446
+
1447
+/**
1448
+ * fence_timeout_start() - Start a timer to check fence-wait timeout
1449
+ *
1450
+ * @cmd: KCPU command queue
1451
+ *
1452
+ * Activate a timer to check whether a fence-wait command in the queue
1453
+ * gets completed  within FENCE_WAIT_TIMEOUT_MS
1454
+ */
1455
+static void fence_timeout_start(struct kbase_kcpu_command_queue *cmd)
1456
+{
1457
+	mod_timer(&cmd->fence_timeout, jiffies + msecs_to_jiffies(FENCE_WAIT_TIMEOUT_MS));
1458
+}
1459
+#endif
12191460 
12201461 /**
12211462  * kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command
....@@ -1236,8 +1477,9 @@
12361477 #else
12371478 	struct dma_fence *fence;
12381479 #endif
1480
+	struct kbase_context *const kctx = kcpu_queue->kctx;
12391481 
1240
-	lockdep_assert_held(&kcpu_queue->kctx->csf.kcpu_queues.lock);
1482
+	lockdep_assert_held(&kcpu_queue->lock);
12411483 
12421484 	if (WARN_ON(!fence_info->fence))
12431485 		return -EINVAL;
....@@ -1251,14 +1493,26 @@
12511493 			&fence_info->fence_cb,
12521494 			kbase_csf_fence_wait_callback);
12531495 
1254
-		KBASE_KTRACE_ADD_CSF_KCPU(kcpu_queue->kctx->kbdev,
1255
-					  FENCE_WAIT_START, kcpu_queue,
1496
+		KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev,
1497
+					  KCPU_FENCE_WAIT_START, kcpu_queue,
12561498 					  fence->context, fence->seqno);
12571499 		fence_status = cb_err;
1258
-		if (cb_err == 0)
1500
+		if (cb_err == 0) {
12591501 			kcpu_queue->fence_wait_processed = true;
1260
-		else if (cb_err == -ENOENT)
1502
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
1503
+			fence_timeout_start(kcpu_queue);
1504
+#endif
1505
+		} else if (cb_err == -ENOENT) {
12611506 			fence_status = dma_fence_get_status(fence);
1507
+			if (!fence_status) {
1508
+				struct kbase_sync_fence_info info;
1509
+
1510
+				kbase_sync_fence_info_get(fence, &info);
1511
+				dev_warn(kctx->kbdev->dev,
1512
+					 "Unexpected status for fence %s of ctx:%d_%d kcpu queue:%u",
1513
+					 info.name, kctx->tgid, kctx->id, kcpu_queue->id);
1514
+			}
1515
+		}
12621516 	}
12631517 
12641518 	/*
....@@ -1271,17 +1525,15 @@
12711525 	 */
12721526 
12731527 	if (fence_status)
1274
-		kbase_kcpu_fence_wait_cancel(kcpu_queue, fence_info);
1528
+		kbasep_kcpu_fence_wait_cancel(kcpu_queue, fence_info);
12751529 
12761530 	return fence_status;
12771531 }
12781532 
1279
-static int kbase_kcpu_fence_wait_prepare(
1280
-		struct kbase_kcpu_command_queue *kcpu_queue,
1281
-		struct base_kcpu_command_fence_info *fence_info,
1282
-		struct kbase_kcpu_command *current_command)
1533
+static int kbase_kcpu_fence_wait_prepare(struct kbase_kcpu_command_queue *kcpu_queue,
1534
+					 struct base_kcpu_command_fence_info *fence_info,
1535
+					 struct kbase_kcpu_command *current_command)
12831536 {
1284
-	struct kbase_context *const kctx = kcpu_queue->kctx;
12851537 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
12861538 	struct fence *fence_in;
12871539 #else
....@@ -1289,10 +1541,9 @@
12891541 #endif
12901542 	struct base_fence fence;
12911543 
1292
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
1544
+	lockdep_assert_held(&kcpu_queue->lock);
12931545 
1294
-	if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence),
1295
-			sizeof(fence)))
1546
+	if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), sizeof(fence)))
12961547 		return -ENOMEM;
12971548 
12981549 	fence_in = sync_file_get_fence(fence.basep.fd);
....@@ -1303,13 +1554,11 @@
13031554 	current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_WAIT;
13041555 	current_command->info.fence.fence = fence_in;
13051556 	current_command->info.fence.kcpu_queue = kcpu_queue;
1306
-
13071557 	return 0;
13081558 }
13091559 
1310
-static int kbase_kcpu_fence_signal_process(
1311
-		struct kbase_kcpu_command_queue *kcpu_queue,
1312
-		struct kbase_kcpu_command_fence_info *fence_info)
1560
+static int kbasep_kcpu_fence_signal_process(struct kbase_kcpu_command_queue *kcpu_queue,
1561
+					    struct kbase_kcpu_command_fence_info *fence_info)
13131562 {
13141563 	struct kbase_context *const kctx = kcpu_queue->kctx;
13151564 	int ret;
....@@ -1320,45 +1569,50 @@
13201569 	ret = dma_fence_signal(fence_info->fence);
13211570 
13221571 	if (unlikely(ret < 0)) {
1323
-		dev_warn(kctx->kbdev->dev,
1324
-			"fence_signal() failed with %d\n", ret);
1572
+		dev_warn(kctx->kbdev->dev, "dma_fence(%d) has been signalled already\n", ret);
1573
+		/* Treated as a success */
1574
+		ret = 0;
13251575 	}
13261576 
1327
-	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_SIGNAL, kcpu_queue,
1577
+	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_FENCE_SIGNAL, kcpu_queue,
13281578 				  fence_info->fence->context,
13291579 				  fence_info->fence->seqno);
13301580 
1331
-	dma_fence_put(fence_info->fence);
1581
+	/* dma_fence refcount needs to be decreased to release it. */
1582
+	kbase_fence_put(fence_info->fence);
13321583 	fence_info->fence = NULL;
13331584 
13341585 	return ret;
13351586 }
13361587 
1337
-static int kbase_kcpu_fence_signal_prepare(
1338
-		struct kbase_kcpu_command_queue *kcpu_queue,
1339
-		struct base_kcpu_command_fence_info *fence_info,
1340
-		struct kbase_kcpu_command *current_command)
1588
+static int kbasep_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_queue,
1589
+					 struct kbase_kcpu_command *current_command,
1590
+					 struct base_fence *fence, struct sync_file **sync_file,
1591
+					 int *fd)
13411592 {
1342
-	struct kbase_context *const kctx = kcpu_queue->kctx;
13431593 #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
13441594 	struct fence *fence_out;
13451595 #else
13461596 	struct dma_fence *fence_out;
13471597 #endif
1348
-	struct base_fence fence;
1349
-	struct sync_file *sync_file;
1598
+	struct kbase_kcpu_dma_fence *kcpu_fence;
13501599 	int ret = 0;
1351
-	int fd;
13521600 
1353
-	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
1601
+	lockdep_assert_held(&kcpu_queue->lock);
13541602 
1355
-	if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence),
1356
-			sizeof(fence)))
1357
-		return -EFAULT;
1358
-
1359
-	fence_out = kzalloc(sizeof(*fence_out), GFP_KERNEL);
1360
-	if (!fence_out)
1603
+	kcpu_fence = kzalloc(sizeof(*kcpu_fence), GFP_KERNEL);
1604
+	if (!kcpu_fence)
13611605 		return -ENOMEM;
1606
+
1607
+	/* Set reference to KCPU metadata and increment refcount */
1608
+	kcpu_fence->metadata = kcpu_queue->metadata;
1609
+	WARN_ON(!kbase_refcount_inc_not_zero(&kcpu_fence->metadata->refcount));
1610
+
1611
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
1612
+	fence_out = (struct fence *)kcpu_fence;
1613
+#else
1614
+	fence_out = (struct dma_fence *)kcpu_fence;
1615
+#endif
13621616 
13631617 	dma_fence_init(fence_out,
13641618 		       &kbase_fence_ops,
....@@ -1376,43 +1630,98 @@
13761630 #endif
13771631 
13781632 	/* create a sync_file fd representing the fence */
1379
-	sync_file = sync_file_create(fence_out);
1380
-	if (!sync_file) {
1381
-#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE)
1382
-		dma_fence_put(fence_out);
1383
-#endif
1633
+	*sync_file = sync_file_create(fence_out);
1634
+	if (!(*sync_file)) {
13841635 		ret = -ENOMEM;
13851636 		goto file_create_fail;
13861637 	}
13871638 
1388
-	fd = get_unused_fd_flags(O_CLOEXEC);
1389
-	if (fd < 0) {
1390
-		ret = fd;
1639
+	*fd = get_unused_fd_flags(O_CLOEXEC);
1640
+	if (*fd < 0) {
1641
+		ret = *fd;
13911642 		goto fd_flags_fail;
13921643 	}
13931644 
1394
-	fd_install(fd, sync_file->file);
1395
-
1396
-	fence.basep.fd = fd;
1645
+	fence->basep.fd = *fd;
13971646 
13981647 	current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL;
13991648 	current_command->info.fence.fence = fence_out;
14001649 
1401
-	if (copy_to_user(u64_to_user_ptr(fence_info->fence), &fence,
1402
-			sizeof(fence))) {
1403
-		ret = -EFAULT;
1404
-		goto fd_flags_fail;
1405
-	}
1406
-
14071650 	return 0;
14081651 
14091652 fd_flags_fail:
1410
-	fput(sync_file->file);
1653
+	fput((*sync_file)->file);
14111654 file_create_fail:
1412
-	dma_fence_put(fence_out);
1655
+	/*
1656
+	 * Upon failure, dma_fence refcount that was increased by
1657
+	 * dma_fence_get() or sync_file_create() needs to be decreased
1658
+	 * to release it.
1659
+	 */
1660
+	kbase_fence_put(fence_out);
1661
+	current_command->info.fence.fence = NULL;
14131662 
14141663 	return ret;
14151664 }
1665
+
1666
+static int kbase_kcpu_fence_signal_prepare(struct kbase_kcpu_command_queue *kcpu_queue,
1667
+					   struct base_kcpu_command_fence_info *fence_info,
1668
+					   struct kbase_kcpu_command *current_command)
1669
+{
1670
+	struct base_fence fence;
1671
+	struct sync_file *sync_file = NULL;
1672
+	int fd;
1673
+	int ret = 0;
1674
+
1675
+	lockdep_assert_held(&kcpu_queue->lock);
1676
+
1677
+	if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), sizeof(fence)))
1678
+		return -EFAULT;
1679
+
1680
+	ret = kbasep_kcpu_fence_signal_init(kcpu_queue, current_command, &fence, &sync_file, &fd);
1681
+	if (ret)
1682
+		return ret;
1683
+
1684
+	if (copy_to_user(u64_to_user_ptr(fence_info->fence), &fence,
1685
+			sizeof(fence))) {
1686
+		ret = -EFAULT;
1687
+		goto fail;
1688
+	}
1689
+
1690
+	/* 'sync_file' pointer can't be safely dereferenced once 'fd' is
1691
+	 * installed, so the install step needs to be done at the last
1692
+	 * before returning success.
1693
+	 */
1694
+	fd_install(fd, sync_file->file);
1695
+	return 0;
1696
+
1697
+fail:
1698
+	fput(sync_file->file);
1699
+	kbase_fence_put(current_command->info.fence.fence);
1700
+	current_command->info.fence.fence = NULL;
1701
+
1702
+	return ret;
1703
+}
1704
+
1705
+int kbase_kcpu_fence_signal_process(struct kbase_kcpu_command_queue *kcpu_queue,
1706
+				    struct kbase_kcpu_command_fence_info *fence_info)
1707
+{
1708
+	if (!kcpu_queue || !fence_info)
1709
+		return -EINVAL;
1710
+
1711
+	return kbasep_kcpu_fence_signal_process(kcpu_queue, fence_info);
1712
+}
1713
+KBASE_EXPORT_TEST_API(kbase_kcpu_fence_signal_process);
1714
+
1715
+int kbase_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_queue,
1716
+				 struct kbase_kcpu_command *current_command,
1717
+				 struct base_fence *fence, struct sync_file **sync_file, int *fd)
1718
+{
1719
+	if (!kcpu_queue || !current_command || !fence || !sync_file || !fd)
1720
+		return -EINVAL;
1721
+
1722
+	return kbasep_kcpu_fence_signal_init(kcpu_queue, current_command, fence, sync_file, fd);
1723
+}
1724
+KBASE_EXPORT_TEST_API(kbase_kcpu_fence_signal_init);
14161725 #endif /* CONFIG_SYNC_FILE */
14171726 
14181727 static void kcpu_queue_process_worker(struct work_struct *data)
....@@ -1420,11 +1729,9 @@
14201729 	struct kbase_kcpu_command_queue *queue = container_of(data,
14211730 				struct kbase_kcpu_command_queue, work);
14221731 
1423
-	mutex_lock(&queue->kctx->csf.kcpu_queues.lock);
1424
-
1732
+	mutex_lock(&queue->lock);
14251733 	kcpu_queue_process(queue, false);
1426
-
1427
-	mutex_unlock(&queue->kctx->csf.kcpu_queues.lock);
1734
+	mutex_unlock(&queue->lock);
14281735 }
14291736 
14301737 static int delete_queue(struct kbase_context *kctx, u32 id)
....@@ -1437,8 +1744,22 @@
14371744 		struct kbase_kcpu_command_queue *queue =
14381745 					kctx->csf.kcpu_queues.array[id];
14391746 
1440
-		KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_DESTROY,
1747
+		KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_DELETE,
14411748 			queue, queue->num_pending_cmds, queue->cqs_wait_count);
1749
+
1750
+		/* Disassociate the queue from the system to prevent further
1751
+		 * submissions. Draining pending commands would be acceptable
1752
+		 * even if a new queue is created using the same ID.
1753
+		 */
1754
+		kctx->csf.kcpu_queues.array[id] = NULL;
1755
+		bitmap_clear(kctx->csf.kcpu_queues.in_use, id, 1);
1756
+
1757
+		mutex_unlock(&kctx->csf.kcpu_queues.lock);
1758
+
1759
+		mutex_lock(&queue->lock);
1760
+
1761
+		/* Metadata struct may outlive KCPU queue.  */
1762
+		kbase_kcpu_dma_fence_meta_put(queue->metadata);
14421763 
14431764 		/* Drain the remaining work for this queue first and go past
14441765 		 * all the waits.
....@@ -1451,22 +1772,22 @@
14511772 		/* All CQS wait commands should have been cleaned up */
14521773 		WARN_ON(queue->cqs_wait_count);
14531774 
1454
-		kctx->csf.kcpu_queues.array[id] = NULL;
1455
-		bitmap_clear(kctx->csf.kcpu_queues.in_use, id, 1);
1456
-
14571775 		/* Fire the tracepoint with the mutex held to enforce correct
14581776 		 * ordering with the summary stream.
14591777 		 */
14601778 		KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE(kctx->kbdev, queue);
14611779 
1462
-		mutex_unlock(&kctx->csf.kcpu_queues.lock);
1780
+		mutex_unlock(&queue->lock);
14631781 
14641782 		cancel_work_sync(&queue->work);
1783
+		destroy_workqueue(queue->wq);
1784
+
1785
+		mutex_destroy(&queue->lock);
14651786 
14661787 		kfree(queue);
14671788 	} else {
1468
-		dev_warn(kctx->kbdev->dev,
1469
-			"Attempt to delete a non-existent KCPU queue\n");
1789
+		dev_dbg(kctx->kbdev->dev,
1790
+			"Attempt to delete a non-existent KCPU queue");
14701791 		mutex_unlock(&kctx->csf.kcpu_queues.lock);
14711792 		err = -EINVAL;
14721793 	}
....@@ -1481,8 +1802,7 @@
14811802 {
14821803 	u8 i;
14831804 
1484
-	KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(
1485
-		kbdev, queue);
1805
+	KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(kbdev, queue);
14861806 	for (i = 0; i < jit_alloc->count; i++) {
14871807 		const u8 id = jit_alloc->info[i].id;
14881808 		const struct kbase_va_region *reg = queue->kctx->jit_alloc[id];
....@@ -1512,26 +1832,24 @@
15121832 	struct kbase_device *kbdev,
15131833 	const struct kbase_kcpu_command_queue *queue)
15141834 {
1515
-	KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(
1516
-		kbdev, queue);
1835
+	KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(kbdev, queue);
15171836 }
15181837 
15191838 static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END(
15201839 	struct kbase_device *kbdev,
15211840 	const struct kbase_kcpu_command_queue *queue)
15221841 {
1523
-	KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END(
1524
-		kbdev, queue);
1842
+	KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END(kbdev, queue);
15251843 }
15261844 
15271845 static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue,
1528
-			bool ignore_waits)
1846
+			       bool drain_queue)
15291847 {
15301848 	struct kbase_device *kbdev = queue->kctx->kbdev;
15311849 	bool process_next = true;
15321850 	size_t i;
15331851 
1534
-	lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock);
1852
+	lockdep_assert_held(&queue->lock);
15351853 
15361854 	for (i = 0; i != queue->num_pending_cmds; ++i) {
15371855 		struct kbase_kcpu_command *cmd =
....@@ -1541,16 +1859,15 @@
15411859 		switch (cmd->type) {
15421860 		case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT:
15431861 			if (!queue->command_started) {
1544
-				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START(
1545
-					kbdev, queue);
1862
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START(kbdev,
1863
+											   queue);
15461864 				queue->command_started = true;
15471865 			}
15481866 
15491867 			status = 0;
15501868 #if IS_ENABLED(CONFIG_SYNC_FILE)
1551
-			if (ignore_waits) {
1552
-				kbase_kcpu_fence_wait_cancel(queue,
1553
-					&cmd->info.fence);
1869
+			if (drain_queue) {
1870
+				kbasep_kcpu_fence_wait_cancel(queue, &cmd->info.fence);
15541871 			} else {
15551872 				status = kbase_kcpu_fence_wait_process(queue,
15561873 					&cmd->info.fence);
....@@ -1575,14 +1892,12 @@
15751892 			}
15761893 			break;
15771894 		case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL:
1578
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START(
1579
-				kbdev, queue);
1895
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START(kbdev, queue);
15801896 
15811897 			status = 0;
15821898 
15831899 #if IS_ENABLED(CONFIG_SYNC_FILE)
1584
-			status = kbase_kcpu_fence_signal_process(
1585
-				queue, &cmd->info.fence);
1900
+			status = kbasep_kcpu_fence_signal_process(queue, &cmd->info.fence);
15861901 
15871902 			if (status < 0)
15881903 				queue->has_error = true;
....@@ -1594,14 +1909,14 @@
15941909 			queue->has_error = true;
15951910 #endif
15961911 
1597
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END(
1598
-				kbdev, queue, status);
1912
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END(kbdev, queue,
1913
+										   status);
15991914 			break;
16001915 		case BASE_KCPU_COMMAND_TYPE_CQS_WAIT:
16011916 			status = kbase_kcpu_cqs_wait_process(kbdev, queue,
16021917 						&cmd->info.cqs_wait);
16031918 
1604
-			if (!status && !ignore_waits) {
1919
+			if (!status && !drain_queue) {
16051920 				process_next = false;
16061921 			} else {
16071922 				/* Either all CQS objects were signaled or
....@@ -1623,7 +1938,7 @@
16231938 			status = kbase_kcpu_cqs_wait_operation_process(kbdev, queue,
16241939 						&cmd->info.cqs_wait_operation);
16251940 
1626
-			if (!status && !ignore_waits) {
1941
+			if (!status && !drain_queue) {
16271942 				process_next = false;
16281943 			} else {
16291944 				/* Either all CQS objects were signaled or
....@@ -1645,35 +1960,36 @@
16451960 			/* Clear the queue's error state */
16461961 			queue->has_error = false;
16471962 
1648
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER(
1649
-				kbdev, queue);
1963
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER(kbdev, queue);
16501964 			break;
16511965 		case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: {
16521966 			struct kbase_ctx_ext_res_meta *meta = NULL;
16531967 
1654
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START(
1655
-				kbdev, queue);
1968
+			if (!drain_queue) {
1969
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START(kbdev,
1970
+											   queue);
16561971 
1657
-			kbase_gpu_vm_lock(queue->kctx);
1658
-			meta = kbase_sticky_resource_acquire(
1659
-				queue->kctx, cmd->info.import.gpu_va);
1660
-			kbase_gpu_vm_unlock(queue->kctx);
1972
+				kbase_gpu_vm_lock(queue->kctx);
1973
+				meta = kbase_sticky_resource_acquire(
1974
+					queue->kctx, cmd->info.import.gpu_va);
1975
+				kbase_gpu_vm_unlock(queue->kctx);
16611976 
1662
-			if (meta == NULL) {
1663
-				queue->has_error = true;
1664
-				dev_warn(kbdev->dev,
1665
-						"failed to map an external resource\n");
1977
+				if (meta == NULL) {
1978
+					queue->has_error = true;
1979
+					dev_dbg(
1980
+						kbdev->dev,
1981
+						"failed to map an external resource");
1982
+				}
1983
+
1984
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END(
1985
+					kbdev, queue, meta ? 0 : 1);
16661986 			}
1667
-
1668
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END(
1669
-				kbdev, queue, meta ? 0 : 1);
16701987 			break;
16711988 		}
16721989 		case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: {
16731990 			bool ret;
16741991 
1675
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START(
1676
-				kbdev, queue);
1992
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START(kbdev, queue);
16771993 
16781994 			kbase_gpu_vm_lock(queue->kctx);
16791995 			ret = kbase_sticky_resource_release(
....@@ -1682,19 +1998,19 @@
16821998 
16831999 			if (!ret) {
16842000 				queue->has_error = true;
1685
-				dev_warn(kbdev->dev,
1686
-						"failed to release the reference. resource not found\n");
2001
+				dev_dbg(kbdev->dev,
2002
+						"failed to release the reference. resource not found");
16872003 			}
16882004 
1689
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END(
1690
-				kbdev, queue, ret ? 0 : 1);
2005
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END(kbdev, queue,
2006
+										   ret ? 0 : 1);
16912007 			break;
16922008 		}
16932009 		case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: {
16942010 			bool ret;
16952011 
1696
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START(
1697
-					kbdev, queue);
2012
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START(kbdev,
2013
+											   queue);
16982014 
16992015 			kbase_gpu_vm_lock(queue->kctx);
17002016 			ret = kbase_sticky_resource_release_force(
....@@ -1703,8 +2019,8 @@
17032019 
17042020 			if (!ret) {
17052021 				queue->has_error = true;
1706
-				dev_warn(kbdev->dev,
1707
-						"failed to release the reference. resource not found\n");
2022
+				dev_dbg(kbdev->dev,
2023
+						"failed to release the reference. resource not found");
17082024 			}
17092025 
17102026 			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END(
....@@ -1713,29 +2029,36 @@
17132029 		}
17142030 		case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC:
17152031 		{
1716
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START(
1717
-				kbdev, queue);
1718
-
1719
-			status = kbase_kcpu_jit_allocate_process(queue, cmd);
1720
-			if (status == -EAGAIN) {
1721
-				process_next = false;
1722
-			} else {
1723
-				if (status != 0)
1724
-					queue->has_error = true;
1725
-
1726
-				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO(
1727
-					kbdev, queue, &cmd->info.jit_alloc,
1728
-					status);
1729
-
2032
+			if (drain_queue) {
2033
+				/* We still need to call this function to clean the JIT alloc info up */
17302034 				kbase_kcpu_jit_allocate_finish(queue, cmd);
1731
-				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(
2035
+			} else {
2036
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START(kbdev,
2037
+											  queue);
2038
+
2039
+				status = kbase_kcpu_jit_allocate_process(queue,
2040
+									 cmd);
2041
+				if (status == -EAGAIN) {
2042
+					process_next = false;
2043
+				} else {
2044
+					if (status != 0)
2045
+						queue->has_error = true;
2046
+
2047
+					KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO(
2048
+						kbdev, queue,
2049
+						&cmd->info.jit_alloc, status);
2050
+
2051
+					kbase_kcpu_jit_allocate_finish(queue,
2052
+								       cmd);
2053
+					KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END(
17322054 						kbdev, queue);
2055
+				}
17332056 			}
2057
+
17342058 			break;
17352059 		}
1736
-		case BASE_KCPU_COMMAND_TYPE_JIT_FREE:
1737
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START(
1738
-				kbdev, queue);
2060
+		case BASE_KCPU_COMMAND_TYPE_JIT_FREE: {
2061
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START(kbdev, queue);
17392062 
17402063 			status = kbase_kcpu_jit_free_process(queue, cmd);
17412064 			if (status)
....@@ -1744,21 +2067,25 @@
17442067 			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END(
17452068 				kbdev, queue);
17462069 			break;
2070
+		}
2071
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
17472072 		case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: {
17482073 			struct kbase_suspend_copy_buffer *sus_buf =
17492074 					cmd->info.suspend_buf_copy.sus_buf;
17502075 
1751
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START(
1752
-				kbdev, queue);
2076
+			if (!drain_queue) {
2077
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START(
2078
+					kbdev, queue);
17532079 
1754
-			status = kbase_csf_queue_group_suspend_process(
2080
+				status = kbase_csf_queue_group_suspend_process(
17552081 					queue->kctx, sus_buf,
17562082 					cmd->info.suspend_buf_copy.group_handle);
1757
-			if (status)
1758
-				queue->has_error = true;
2083
+				if (status)
2084
+					queue->has_error = true;
17592085 
1760
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END(
1761
-				kbdev, queue, status);
2086
+				KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END(
2087
+					kbdev, queue, status);
2088
+			}
17622089 
17632090 			if (!sus_buf->cpu_alloc) {
17642091 				int i;
....@@ -1768,36 +2095,18 @@
17682095 			} else {
17692096 				kbase_mem_phy_alloc_kernel_unmapped(
17702097 					sus_buf->cpu_alloc);
1771
-				kbase_mem_phy_alloc_put(sus_buf->cpu_alloc);
2098
+				kbase_mem_phy_alloc_put(
2099
+					sus_buf->cpu_alloc);
17722100 			}
17732101 
17742102 			kfree(sus_buf->pages);
17752103 			kfree(sus_buf);
17762104 			break;
17772105 		}
1778
-#if MALI_UNIT_TEST
1779
-		case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: {
1780
-			u64 time = ktime_get_raw_ns();
1781
-			void *target_page = kmap(*cmd->info.sample_time.page);
1782
-
1783
-			if (target_page) {
1784
-				memcpy(target_page +
1785
-					       cmd->info.sample_time.page_offset,
1786
-				       &time, sizeof(time));
1787
-				kunmap(*cmd->info.sample_time.page);
1788
-			} else {
1789
-				dev_warn(kbdev->dev,
1790
-					 "Could not kmap target page\n");
1791
-				queue->has_error = true;
1792
-			}
1793
-			put_page(*cmd->info.sample_time.page);
1794
-			kfree(cmd->info.sample_time.page);
1795
-			break;
1796
-		}
1797
-#endif /* MALI_UNIT_TEST */
2106
+#endif
17982107 		default:
1799
-			dev_warn(kbdev->dev,
1800
-				"Unrecognized command type\n");
2108
+			dev_dbg(kbdev->dev,
2109
+				"Unrecognized command type");
18012110 			break;
18022111 		} /* switch */
18032112 
....@@ -1835,12 +2144,12 @@
18352144 
18362145 	switch (cmd->type) {
18372146 	case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT:
1838
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT(
1839
-			kbdev, queue, cmd->info.fence.fence);
2147
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT(kbdev, queue,
2148
+								     cmd->info.fence.fence);
18402149 		break;
18412150 	case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL:
1842
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL(
1843
-			kbdev, queue, cmd->info.fence.fence);
2151
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL(kbdev, queue,
2152
+								       cmd->info.fence.fence);
18442153 		break;
18452154 	case BASE_KCPU_COMMAND_TYPE_CQS_WAIT:
18462155 	{
....@@ -1862,32 +2171,48 @@
18622171 		unsigned int i;
18632172 
18642173 		for (i = 0; i < cmd->info.cqs_set.nr_objs; i++) {
1865
-			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET(
1866
-				kbdev, queue, sets[i].addr);
2174
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET(kbdev, queue,
2175
+									  sets[i].addr);
18672176 		}
18682177 		break;
18692178 	}
18702179 	case BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION:
18712180 	{
1872
-		/* GPUCORE-28172 RDT to review */
2181
+		const struct base_cqs_wait_operation_info *waits =
2182
+			cmd->info.cqs_wait_operation.objs;
2183
+		u32 inherit_err_flags = cmd->info.cqs_wait_operation.inherit_err_flags;
2184
+		unsigned int i;
2185
+
2186
+		for (i = 0; i < cmd->info.cqs_wait_operation.nr_objs; i++) {
2187
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION(
2188
+				kbdev, queue, waits[i].addr, waits[i].val,
2189
+				waits[i].operation, waits[i].data_type,
2190
+				(inherit_err_flags & ((uint32_t)1 << i)) ? 1 : 0);
2191
+		}
18732192 		break;
18742193 	}
18752194 	case BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION:
18762195 	{
1877
-		/* GPUCORE-28172 RDT to review */
2196
+		const struct base_cqs_set_operation_info *sets = cmd->info.cqs_set_operation.objs;
2197
+		unsigned int i;
2198
+
2199
+		for (i = 0; i < cmd->info.cqs_set_operation.nr_objs; i++) {
2200
+			KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION(
2201
+				kbdev, queue, sets[i].addr, sets[i].val,
2202
+				sets[i].operation, sets[i].data_type);
2203
+		}
18782204 		break;
18792205 	}
18802206 	case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER:
1881
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER(kbdev,
1882
-									queue);
2207
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER(kbdev, queue);
18832208 		break;
18842209 	case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT:
1885
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT(
1886
-			kbdev, queue, cmd->info.import.gpu_va);
2210
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT(kbdev, queue,
2211
+								     cmd->info.import.gpu_va);
18872212 		break;
18882213 	case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT:
1889
-		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT(
1890
-			kbdev, queue, cmd->info.import.gpu_va);
2214
+		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT(kbdev, queue,
2215
+								       cmd->info.import.gpu_va);
18912216 		break;
18922217 	case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE:
18932218 		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE(
....@@ -1897,50 +2222,41 @@
18972222 	{
18982223 		u8 i;
18992224 
1900
-		KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC(
1901
-			kbdev, queue);
2225
+		KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC(kbdev, queue);
19022226 		for (i = 0; i < cmd->info.jit_alloc.count; i++) {
19032227 			const struct base_jit_alloc_info *info =
19042228 				&cmd->info.jit_alloc.info[i];
19052229 
19062230 			KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC(
1907
-				kbdev, queue, info->gpu_alloc_addr,
1908
-				info->va_pages, info->commit_pages,
1909
-				info->extension, info->id, info->bin_id,
1910
-				info->max_allocations, info->flags,
1911
-				info->usage_id);
2231
+				kbdev, queue, info->gpu_alloc_addr, info->va_pages,
2232
+				info->commit_pages, info->extension, info->id, info->bin_id,
2233
+				info->max_allocations, info->flags, info->usage_id);
19122234 		}
1913
-		KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC(
1914
-			kbdev, queue);
2235
+		KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC(kbdev, queue);
19152236 		break;
19162237 	}
19172238 	case BASE_KCPU_COMMAND_TYPE_JIT_FREE:
19182239 	{
19192240 		u8 i;
19202241 
1921
-		KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE(
1922
-			kbdev, queue);
2242
+		KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE(kbdev, queue);
19232243 		for (i = 0; i < cmd->info.jit_free.count; i++) {
19242244 			KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE(
19252245 				kbdev, queue, cmd->info.jit_free.ids[i]);
19262246 		}
1927
-		KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE(
1928
-			kbdev, queue);
2247
+		KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE(kbdev, queue);
19292248 		break;
19302249 	}
2250
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
19312251 	case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND:
19322252 		KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND(
19332253 			kbdev, queue, cmd->info.suspend_buf_copy.sus_buf,
19342254 			cmd->info.suspend_buf_copy.group_handle);
19352255 		break;
1936
-#if MALI_UNIT_TEST
1937
-	case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME:
1938
-		/*
1939
-		 * This is test-only KCPU command, no need to have a timeline
1940
-		 * entry
1941
-		 */
2256
+#endif
2257
+	default:
2258
+		dev_dbg(kbdev->dev, "Unknown command type %u", cmd->type);
19422259 		break;
1943
-#endif /* MALI_UNIT_TEST */
19442260 	}
19452261 }
19462262 
....@@ -1954,9 +2270,11 @@
19542270 
19552271 	/* The offset to the first command that is being processed or yet to
19562272 	 * be processed is of u8 type, so the number of commands inside the
1957
-	 * queue cannot be more than 256.
2273
+	 * queue cannot be more than 256. The current implementation expects
2274
+	 * exactly 256, any other size will require the addition of wrapping
2275
+	 * logic.
19582276 	 */
1959
-	BUILD_BUG_ON(KBASEP_KCPU_QUEUE_SIZE > 256);
2277
+	BUILD_BUG_ON(KBASEP_KCPU_QUEUE_SIZE != 256);
19602278 
19612279 	/* Whilst the backend interface allows enqueueing multiple commands in
19622280 	 * a single operation, the Base interface does not expose any mechanism
....@@ -1966,19 +2284,35 @@
19662284 	 * in the set.
19672285 	 */
19682286 	if (enq->nr_commands != 1) {
1969
-		dev_err(kctx->kbdev->dev,
1970
-			"More than one commands enqueued\n");
2287
+		dev_dbg(kctx->kbdev->dev,
2288
+			"More than one commands enqueued");
19712289 		return -EINVAL;
19722290 	}
19732291 
2292
+	/* There might be a race between one thread trying to enqueue commands to the queue
2293
+	 * and other thread trying to delete the same queue.
2294
+	 * This racing could lead to use-after-free problem by enqueuing thread if
2295
+	 * resources for the queue has already been freed by deleting thread.
2296
+	 *
2297
+	 * To prevent the issue, two mutexes are acquired/release asymmetrically as follows.
2298
+	 *
2299
+	 * Lock A (kctx mutex)
2300
+	 * Lock B (queue mutex)
2301
+	 * Unlock A
2302
+	 * Unlock B
2303
+	 *
2304
+	 * With the kctx mutex being held, enqueuing thread will check the queue
2305
+	 * and will return error code if the queue had already been deleted.
2306
+	 */
19742307 	mutex_lock(&kctx->csf.kcpu_queues.lock);
1975
-
1976
-	if (!kctx->csf.kcpu_queues.array[enq->id]) {
1977
-		ret = -EINVAL;
1978
-		goto out;
1979
-	}
1980
-
19812308 	queue = kctx->csf.kcpu_queues.array[enq->id];
2309
+	if (queue == NULL) {
2310
+		dev_dbg(kctx->kbdev->dev, "Invalid KCPU queue (id:%u)", enq->id);
2311
+		mutex_unlock(&kctx->csf.kcpu_queues.lock);
2312
+		return -EINVAL;
2313
+	}
2314
+	mutex_lock(&queue->lock);
2315
+	mutex_unlock(&kctx->csf.kcpu_queues.lock);
19822316 
19832317 	if (kcpu_queue_get_space(queue) < enq->nr_commands) {
19842318 		ret = -EBUSY;
....@@ -1993,7 +2327,7 @@
19932327 	 * for the possibility to roll back.
19942328 	 */
19952329 
1996
-	for (i = 0; (i != enq->nr_commands) && !ret; ++i, ++kctx->csf.kcpu_queues.num_cmds) {
2330
+	for (i = 0; (i != enq->nr_commands) && !ret; ++i) {
19972331 		struct kbase_kcpu_command *kcpu_cmd =
19982332 			&queue->commands[(u8)(queue->start_offset + queue->num_pending_cmds + i)];
19992333 		struct base_kcpu_command command;
....@@ -2016,7 +2350,7 @@
20162350 			}
20172351 		}
20182352 
2019
-		kcpu_cmd->enqueue_ts = kctx->csf.kcpu_queues.num_cmds;
2353
+		kcpu_cmd->enqueue_ts = atomic64_inc_return(&kctx->csf.kcpu_queues.cmd_seq_num);
20202354 		switch (command.type) {
20212355 		case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT:
20222356 #if IS_ENABLED(CONFIG_SYNC_FILE)
....@@ -2076,45 +2410,16 @@
20762410 			ret = kbase_kcpu_jit_free_prepare(queue,
20772411 					&command.info.jit_free, kcpu_cmd);
20782412 			break;
2413
+#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST
20792414 		case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND:
20802415 			ret = kbase_csf_queue_group_suspend_prepare(queue,
20812416 					&command.info.suspend_buf_copy,
20822417 					kcpu_cmd);
20832418 			break;
2084
-#if MALI_UNIT_TEST
2085
-		case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: {
2086
-			int const page_cnt = 1;
2087
-
2088
-			kcpu_cmd->type = BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME;
2089
-			kcpu_cmd->info.sample_time.page_addr =
2090
-				command.info.sample_time.time & PAGE_MASK;
2091
-			kcpu_cmd->info.sample_time.page_offset =
2092
-				command.info.sample_time.time & ~PAGE_MASK;
2093
-			kcpu_cmd->info.sample_time.page = kcalloc(
2094
-				page_cnt, sizeof(struct page *), GFP_KERNEL);
2095
-			if (!kcpu_cmd->info.sample_time.page) {
2096
-				ret = -ENOMEM;
2097
-			} else {
2098
-				int pinned_pages = get_user_pages_fast(
2099
-					kcpu_cmd->info.sample_time.page_addr,
2100
-					page_cnt, 1,
2101
-					kcpu_cmd->info.sample_time.page);
2102
-
2103
-				if (pinned_pages < 0) {
2104
-					ret = pinned_pages;
2105
-					kfree(kcpu_cmd->info.sample_time.page);
2106
-				} else if (pinned_pages != page_cnt) {
2107
-					ret = -EINVAL;
2108
-					kfree(kcpu_cmd->info.sample_time.page);
2109
-				}
2110
-			}
2111
-
2112
-			break;
2113
-		}
2114
-#endif /* MALI_UNIT_TEST */
2419
+#endif
21152420 		default:
2116
-			dev_warn(queue->kctx->kbdev->dev,
2117
-				"Unknown command type %u\n", command.type);
2421
+			dev_dbg(queue->kctx->kbdev->dev,
2422
+				"Unknown command type %u", command.type);
21182423 			ret = -EINVAL;
21192424 			break;
21202425 		}
....@@ -2135,13 +2440,10 @@
21352440 
21362441 		queue->num_pending_cmds += enq->nr_commands;
21372442 		kcpu_queue_process(queue, false);
2138
-	} else {
2139
-		/* Roll back the number of enqueued commands */
2140
-		kctx->csf.kcpu_queues.num_cmds -= i;
21412443 	}
21422444 
21432445 out:
2144
-	mutex_unlock(&kctx->csf.kcpu_queues.lock);
2446
+	mutex_unlock(&queue->lock);
21452447 
21462448 	return ret;
21472449 }
....@@ -2155,14 +2457,9 @@
21552457 	for (idx = 0; idx < KBASEP_MAX_KCPU_QUEUES; ++idx)
21562458 		kctx->csf.kcpu_queues.array[idx] = NULL;
21572459 
2158
-	kctx->csf.kcpu_queues.wq = alloc_workqueue("mali_kbase_csf_kcpu",
2159
-					WQ_UNBOUND | WQ_HIGHPRI, 0);
2160
-	if (!kctx->csf.kcpu_queues.wq)
2161
-		return -ENOMEM;
2162
-
21632460 	mutex_init(&kctx->csf.kcpu_queues.lock);
21642461 
2165
-	kctx->csf.kcpu_queues.num_cmds = 0;
2462
+	atomic64_set(&kctx->csf.kcpu_queues.cmd_seq_num, 0);
21662463 
21672464 	return 0;
21682465 }
....@@ -2180,9 +2477,9 @@
21802477 			(void)delete_queue(kctx, id);
21812478 	}
21822479 
2183
-	destroy_workqueue(kctx->csf.kcpu_queues.wq);
21842480 	mutex_destroy(&kctx->csf.kcpu_queues.lock);
21852481 }
2482
+KBASE_EXPORT_TEST_API(kbase_csf_kcpu_queue_context_term);
21862483 
21872484 int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx,
21882485 			struct kbase_ioctl_kcpu_queue_delete *del)
....@@ -2195,8 +2492,11 @@
21952492 {
21962493 	struct kbase_kcpu_command_queue *queue;
21972494 	int idx;
2495
+	int n;
21982496 	int ret = 0;
2199
-
2497
+#if IS_ENABLED(CONFIG_SYNC_FILE)
2498
+	struct kbase_kcpu_dma_fence_meta *metadata;
2499
+#endif
22002500 	/* The queue id is of u8 type and we use the index of the kcpu_queues
22012501 	 * array as an id, so the number of elements in the array can't be
22022502 	 * more than 256.
....@@ -2224,8 +2524,17 @@
22242524 		goto out;
22252525 	}
22262526 
2527
+	queue->wq = alloc_workqueue("mali_kbase_csf_kcpu_wq_%i", WQ_UNBOUND | WQ_HIGHPRI, 0, idx);
2528
+	if (queue->wq == NULL) {
2529
+		kfree(queue);
2530
+		ret = -ENOMEM;
2531
+
2532
+		goto out;
2533
+	}
2534
+
22272535 	bitmap_set(kctx->csf.kcpu_queues.in_use, idx, 1);
22282536 	kctx->csf.kcpu_queues.array[idx] = queue;
2537
+	mutex_init(&queue->lock);
22292538 	queue->kctx = kctx;
22302539 	queue->start_offset = 0;
22312540 	queue->num_pending_cmds = 0;
....@@ -2233,7 +2542,31 @@
22332542 	queue->fence_context = dma_fence_context_alloc(1);
22342543 	queue->fence_seqno = 0;
22352544 	queue->fence_wait_processed = false;
2236
-#endif
2545
+
2546
+	metadata = kzalloc(sizeof(*metadata), GFP_KERNEL);
2547
+	if (!metadata) {
2548
+		destroy_workqueue(queue->wq);
2549
+		kfree(queue);
2550
+		ret = -ENOMEM;
2551
+		goto out;
2552
+	}
2553
+
2554
+	metadata->kbdev = kctx->kbdev;
2555
+	metadata->kctx_id = kctx->id;
2556
+	n = snprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%d-%d_%d-%lld-kcpu",
2557
+		     kctx->kbdev->id, kctx->tgid, kctx->id, queue->fence_context);
2558
+	if (WARN_ON(n >= MAX_TIMELINE_NAME)) {
2559
+		destroy_workqueue(queue->wq);
2560
+		kfree(queue);
2561
+		kfree(metadata);
2562
+		ret = -EINVAL;
2563
+		goto out;
2564
+	}
2565
+
2566
+	kbase_refcount_set(&metadata->refcount, 1);
2567
+	queue->metadata = metadata;
2568
+	atomic_inc(&kctx->kbdev->live_fence_metadata);
2569
+#endif /* CONFIG_SYNC_FILE */
22372570 	queue->enqueue_failed = false;
22382571 	queue->command_started = false;
22392572 	INIT_LIST_HEAD(&queue->jit_blocked);
....@@ -2246,13 +2579,17 @@
22462579 	/* Fire the tracepoint with the mutex held to enforce correct ordering
22472580 	 * with the summary stream.
22482581 	 */
2249
-	KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE(
2250
-		kctx->kbdev, queue, kctx->id, queue->num_pending_cmds);
2582
+	KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE(kctx->kbdev, queue, queue->id, kctx->id,
2583
+					      queue->num_pending_cmds);
22512584 
2252
-	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_NEW, queue,
2585
+	KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_CREATE, queue,
22532586 		queue->fence_context, 0);
2587
+#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG
2588
+	kbase_timer_setup(&queue->fence_timeout, fence_timeout_callback);
2589
+#endif
22542590 out:
22552591 	mutex_unlock(&kctx->csf.kcpu_queues.lock);
22562592 
22572593 	return ret;
22582594 }
2595
+KBASE_EXPORT_TEST_API(kbase_csf_kcpu_queue_new);