forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
....@@ -21,8 +21,10 @@
2121  *
2222  * Authors: Alex Deucher
2323  */
24
+
2425 #include <linux/firmware.h>
25
-#include <drm/drmP.h>
26
+#include <linux/module.h>
27
+
2628 #include "amdgpu.h"
2729 #include "amdgpu_ucode.h"
2830 #include "amdgpu_trace.h"
....@@ -198,7 +200,7 @@
198200 
199201 static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
200202 {
201
-	struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);
203
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
202204 	int i;
203205 
204206 	for (i = 0; i < count; i++)
....@@ -218,13 +220,15 @@
218220  * Schedule an IB in the DMA ring (CIK).
219221  */
220222 static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,
223
+				  struct amdgpu_job *job,
221224 				  struct amdgpu_ib *ib,
222
-				  unsigned vmid, bool ctx_switch)
225
+				  uint32_t flags)
223226 {
227
+	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
224228 	u32 extra_bits = vmid & 0xf;
225229 
226230 	/* IB packet must end on a 8 DW boundary */
227
-	cik_sdma_ring_insert_nop(ring, (12 - (lower_32_bits(ring->wptr) & 7)) % 8);
231
+	cik_sdma_ring_insert_nop(ring, (4 - lower_32_bits(ring->wptr)) & 7);
228232 
229233 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
230234 	amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
....@@ -316,8 +320,6 @@
316320 		WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
317321 		WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], 0);
318322 	}
319
-	sdma0->ready = false;
320
-	sdma1->ready = false;
321323 }
322324 
323325 /**
....@@ -494,18 +496,16 @@
494496 		/* enable DMA IBs */
495497 		WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);
496498 
497
-		ring->ready = true;
499
+		ring->sched.ready = true;
498500 	}
499501 
500502 	cik_sdma_enable(adev, true);
501503 
502504 	for (i = 0; i < adev->sdma.num_instances; i++) {
503505 		ring = &adev->sdma.instance[i].ring;
504
-		r = amdgpu_ring_test_ring(ring);
505
-		if (r) {
506
-			ring->ready = false;
506
+		r = amdgpu_ring_test_helper(ring);
507
+		if (r)
507508 			return r;
508
-		}
509509 
510510 		if (adev->mman.buffer_funcs_ring == ring)
511511 			amdgpu_ttm_set_buffer_funcs_status(adev, true);
....@@ -618,21 +618,17 @@
618618 	u64 gpu_addr;
619619 
620620 	r = amdgpu_device_wb_get(adev, &index);
621
-	if (r) {
622
-		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
621
+	if (r)
623622 		return r;
624
-	}
625623 
626624 	gpu_addr = adev->wb.gpu_addr + (index * 4);
627625 	tmp = 0xCAFEDEAD;
628626 	adev->wb.wb[index] = cpu_to_le32(tmp);
629627 
630628 	r = amdgpu_ring_alloc(ring, 5);
631
-	if (r) {
632
-		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
633
-		amdgpu_device_wb_free(adev, index);
634
-		return r;
635
-	}
629
+	if (r)
630
+		goto error_free_wb;
631
+
636632 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
637633 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
638634 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
....@@ -644,18 +640,14 @@
644640 		tmp = le32_to_cpu(adev->wb.wb[index]);
645641 		if (tmp == 0xDEADBEEF)
646642 			break;
647
-		DRM_UDELAY(1);
643
+		udelay(1);
648644 	}
649645 
650
-	if (i < adev->usec_timeout) {
651
-		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
652
-	} else {
653
-		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
654
-			  ring->idx, tmp);
655
-		r = -EINVAL;
656
-	}
646
+	if (i >= adev->usec_timeout)
647
+		r = -ETIMEDOUT;
648
+
649
+error_free_wb:
657650 	amdgpu_device_wb_free(adev, index);
658
-
659651 	return r;
660652 }
661653 
....@@ -678,20 +670,17 @@
678670 	long r;
679671 
680672 	r = amdgpu_device_wb_get(adev, &index);
681
-	if (r) {
682
-		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
673
+	if (r)
683674 		return r;
684
-	}
685675 
686676 	gpu_addr = adev->wb.gpu_addr + (index * 4);
687677 	tmp = 0xCAFEDEAD;
688678 	adev->wb.wb[index] = cpu_to_le32(tmp);
689679 	memset(&ib, 0, sizeof(ib));
690
-	r = amdgpu_ib_get(adev, NULL, 256, &ib);
691
-	if (r) {
692
-		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
680
+	r = amdgpu_ib_get(adev, NULL, 256,
681
+					AMDGPU_IB_POOL_DIRECT, &ib);
682
+	if (r)
693683 		goto err0;
694
-	}
695684 
696685 	ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,
697686 				SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
....@@ -706,21 +695,16 @@
706695 
707696 	r = dma_fence_wait_timeout(f, false, timeout);
708697 	if (r == 0) {
709
-		DRM_ERROR("amdgpu: IB test timed out\n");
710698 		r = -ETIMEDOUT;
711699 		goto err1;
712700 	} else if (r < 0) {
713
-		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
714701 		goto err1;
715702 	}
716703 	tmp = le32_to_cpu(adev->wb.wb[index]);
717
-	if (tmp == 0xDEADBEEF) {
718
-		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
704
+	if (tmp == 0xDEADBEEF)
719705 		r = 0;
720
-	} else {
721
-		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
706
+	else
722707 		r = -EINVAL;
723
-	}
724708 
725709 err1:
726710 	amdgpu_ib_free(adev, &ib, NULL);
....@@ -822,11 +806,11 @@
822806  */
823807 static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
824808 {
825
-	struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);
809
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
826810 	u32 pad_count;
827811 	int i;
828812 
829
-	pad_count = (8 - (ib->length_dw & 0x7)) % 8;
813
+	pad_count = (-ib->length_dw) & 7;
830814 	for (i = 0; i < pad_count; i++)
831815 		if (sdma && sdma->burst_nop && (i == 0))
832816 			ib->ptr[ib->length_dw++] =
....@@ -970,19 +954,19 @@
970954 	}
971955 
972956 	/* SDMA trap event */
973
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 224,
957
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224,
974958 			      &adev->sdma.trap_irq);
975959 	if (r)
976960 		return r;
977961 
978962 	/* SDMA Privileged inst */
979
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 241,
963
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 241,
980964 			      &adev->sdma.illegal_inst_irq);
981965 	if (r)
982966 		return r;
983967 
984968 	/* SDMA Privileged inst */
985
-	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 247,
969
+	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 247,
986970 			      &adev->sdma.illegal_inst_irq);
987971 	if (r)
988972 		return r;
....@@ -994,8 +978,9 @@
994978 		r = amdgpu_ring_init(adev, ring, 1024,
995979 				     &adev->sdma.trap_irq,
996980 				     (i == 0) ?
997
-				     AMDGPU_SDMA_IRQ_TRAP0 :
998
-				     AMDGPU_SDMA_IRQ_TRAP1);
981
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
982
+				     AMDGPU_SDMA_IRQ_INSTANCE1,
983
+				     AMDGPU_RING_PRIO_DEFAULT);
999984 		if (r)
1000985 			return r;
1001986 	}
....@@ -1128,7 +1113,7 @@
11281113 	u32 sdma_cntl;
11291114 
11301115 	switch (type) {
1131
-	case AMDGPU_SDMA_IRQ_TRAP0:
1116
+	case AMDGPU_SDMA_IRQ_INSTANCE0:
11321117 		switch (state) {
11331118 		case AMDGPU_IRQ_STATE_DISABLE:
11341119 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
....@@ -1144,7 +1129,7 @@
11441129 			break;
11451130 		}
11461131 		break;
1147
-	case AMDGPU_SDMA_IRQ_TRAP1:
1132
+	case AMDGPU_SDMA_IRQ_INSTANCE1:
11481133 		switch (state) {
11491134 		case AMDGPU_IRQ_STATE_DISABLE:
11501135 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
....@@ -1211,8 +1196,11 @@
12111196 					     struct amdgpu_irq_src *source,
12121197 					     struct amdgpu_iv_entry *entry)
12131198 {
1199
+	u8 instance_id;
1200
+
12141201 	DRM_ERROR("Illegal instruction in SDMA command stream\n");
1215
-	schedule_work(&adev->reset_work);
1202
+	instance_id = (entry->ring_id & 0x3) >> 0;
1203
+	drm_sched_fault(&adev->sdma.instance[instance_id].ring.sched);
12161204 	return 0;
12171205 }
12181206 
....@@ -1322,7 +1310,8 @@
13221310 static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib,
13231311 				      uint64_t src_offset,
13241312 				      uint64_t dst_offset,
1325
-				      uint32_t byte_count)
1313
+				      uint32_t byte_count,
1314
+				      bool tmz)
13261315 {
13271316 	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0);
13281317 	ib->ptr[ib->length_dw++] = byte_count;
....@@ -1367,10 +1356,8 @@
13671356 
13681357 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
13691358 {
1370
-	if (adev->mman.buffer_funcs == NULL) {
1371
-		adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
1372
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1373
-	}
1359
+	adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
1360
+	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
13741361 }
13751362 
13761363 static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
....@@ -1385,14 +1372,12 @@
13851372 {
13861373 	unsigned i;
13871374 
1388
-	if (adev->vm_manager.vm_pte_funcs == NULL) {
1389
-		adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
1390
-		for (i = 0; i < adev->sdma.num_instances; i++)
1391
-			adev->vm_manager.vm_pte_rings[i] =
1392
-				&adev->sdma.instance[i].ring;
1393
-
1394
-		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
1375
+	adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
1376
+	for (i = 0; i < adev->sdma.num_instances; i++) {
1377
+		adev->vm_manager.vm_pte_scheds[i] =
1378
+			&adev->sdma.instance[i].ring.sched;
13951379 	}
1380
+	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
13961381 }
13971382 
13981383 const struct amdgpu_ip_block_version cik_sdma_ip_block =