forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390 
 kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
....@@ -28,8 +28,9 @@
2828  */
2929 #include <linux/seq_file.h>
3030 #include <linux/slab.h>
31
+#include <linux/uaccess.h>
3132 #include <linux/debugfs.h>
32
-#include <drm/drmP.h>
33
+
3334 #include <drm/amdgpu_drm.h>
3435 #include "amdgpu.h"
3536 #include "atom.h"
....@@ -47,9 +48,6 @@
4748  * wptr.  The GPU then starts fetching commands and executes
4849  * them until the pointers are equal again.
4950  */
50
-static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
51
-				    struct amdgpu_ring *ring);
52
-static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring);
5351 
5452 /**
5553  * amdgpu_ring_alloc - allocate space on the ring buffer
....@@ -135,9 +133,6 @@
135133 
136134 	if (ring->funcs->end_use)
137135 		ring->funcs->end_use(ring);
138
-
139
-	if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
140
-		amdgpu_ring_lru_touch(ring->adev, ring);
141136 }
142137 
143138 /**
....@@ -156,76 +151,6 @@
156151 }
157152 
158153 /**
159
- * amdgpu_ring_priority_put - restore a ring's priority
160
- *
161
- * @ring: amdgpu_ring structure holding the information
162
- * @priority: target priority
163
- *
164
- * Release a request for executing at @priority
165
- */
166
-void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
167
-			      enum drm_sched_priority priority)
168
-{
169
-	int i;
170
-
171
-	if (!ring->funcs->set_priority)
172
-		return;
173
-
174
-	if (atomic_dec_return(&ring->num_jobs[priority]) > 0)
175
-		return;
176
-
177
-	/* no need to restore if the job is already at the lowest priority */
178
-	if (priority == DRM_SCHED_PRIORITY_NORMAL)
179
-		return;
180
-
181
-	mutex_lock(&ring->priority_mutex);
182
-	/* something higher prio is executing, no need to decay */
183
-	if (ring->priority > priority)
184
-		goto out_unlock;
185
-
186
-	/* decay priority to the next level with a job available */
187
-	for (i = priority; i >= DRM_SCHED_PRIORITY_MIN; i--) {
188
-		if (i == DRM_SCHED_PRIORITY_NORMAL
189
-				|| atomic_read(&ring->num_jobs[i])) {
190
-			ring->priority = i;
191
-			ring->funcs->set_priority(ring, i);
192
-			break;
193
-		}
194
-	}
195
-
196
-out_unlock:
197
-	mutex_unlock(&ring->priority_mutex);
198
-}
199
-
200
-/**
201
- * amdgpu_ring_priority_get - change the ring's priority
202
- *
203
- * @ring: amdgpu_ring structure holding the information
204
- * @priority: target priority
205
- *
206
- * Request a ring's priority to be raised to @priority (refcounted).
207
- */
208
-void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
209
-			      enum drm_sched_priority priority)
210
-{
211
-	if (!ring->funcs->set_priority)
212
-		return;
213
-
214
-	if (atomic_inc_return(&ring->num_jobs[priority]) <= 0)
215
-		return;
216
-
217
-	mutex_lock(&ring->priority_mutex);
218
-	if (priority <= ring->priority)
219
-		goto out_unlock;
220
-
221
-	ring->priority = priority;
222
-	ring->funcs->set_priority(ring, priority);
223
-
224
-out_unlock:
225
-	mutex_unlock(&ring->priority_mutex);
226
-}
227
-
228
-/**
229154  * amdgpu_ring_init - init driver ring struct.
230155  *
231156  * @adev: amdgpu_device pointer
....@@ -237,11 +162,13 @@
237162  * Returns 0 on success, error on failure.
238163  */
239164 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
240
-		     unsigned max_dw, struct amdgpu_irq_src *irq_src,
241
-		     unsigned irq_type)
165
+		     unsigned int max_dw, struct amdgpu_irq_src *irq_src,
166
+		     unsigned int irq_type, unsigned int hw_prio)
242167 {
243168 	int r, i;
244169 	int sched_hw_submission = amdgpu_sched_hw_submission;
170
+	u32 *num_sched;
171
+	u32 hw_ip;
245172 
246173 	/* Set the hw submission limit higher for KIQ because
247174 	 * it's used for a number of gfx/compute tasks by both
....@@ -251,6 +178,8 @@
251178 	 */
252179 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
253180 		sched_hw_submission = max(sched_hw_submission, 256);
181
+	else if (ring == &adev->sdma.instance[0].page)
182
+		sched_hw_submission = 256;
254183 
255184 	if (ring->adev == NULL) {
256185 		if (adev->num_rings >= AMDGPU_MAX_RINGS)
....@@ -281,6 +210,16 @@
281210 		dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
282211 		return r;
283212 	}
213
+
214
+	r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs);
215
+	if (r) {
216
+		dev_err(adev->dev,
217
+			"(%d) ring trail_fence_offs wb alloc failed\n", r);
218
+		return r;
219
+	}
220
+	ring->trail_fence_gpu_addr =
221
+		adev->wb.gpu_addr + (ring->trail_fence_offs * 4);
222
+	ring->trail_fence_cpu_addr = &adev->wb.wb[ring->trail_fence_offs];
284223 
285224 	r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);
286225 	if (r) {
....@@ -320,15 +259,16 @@
320259 	ring->max_dw = max_dw;
321260 	ring->priority = DRM_SCHED_PRIORITY_NORMAL;
322261 	mutex_init(&ring->priority_mutex);
323
-	INIT_LIST_HEAD(&ring->lru_list);
324
-	amdgpu_ring_lru_touch(adev, ring);
325262 
326
-	for (i = 0; i < DRM_SCHED_PRIORITY_MAX; ++i)
327
-		atomic_set(&ring->num_jobs[i], 0);
328
-
329
-	if (amdgpu_debugfs_ring_init(adev, ring)) {
330
-		DRM_ERROR("Failed to register debugfs file for rings !\n");
263
+	if (!ring->no_scheduler) {
264
+		hw_ip = ring->funcs->type;
265
+		num_sched = &adev->gpu_sched[hw_ip][hw_prio].num_scheds;
266
+		adev->gpu_sched[hw_ip][hw_prio].sched[(*num_sched)++] =
267
+			&ring->sched;
331268 	}
269
+
270
+	for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; ++i)
271
+		atomic_set(&ring->num_jobs[i], 0);
332272 
333273 	return 0;
334274 }
....@@ -343,11 +283,12 @@
343283  */
344284 void amdgpu_ring_fini(struct amdgpu_ring *ring)
345285 {
346
-	ring->ready = false;
347286 
348287 	/* Not to finish a ring which is not initialized */
349288 	if (!(ring->adev) || !(ring->adev->rings[ring->idx]))
350289 		return;
290
+
291
+	ring->sched.ready = false;
351292 
352293 	amdgpu_device_wb_free(ring->adev, ring->rptr_offs);
353294 	amdgpu_device_wb_free(ring->adev, ring->wptr_offs);
....@@ -359,106 +300,11 @@
359300 			      &ring->gpu_addr,
360301 			      (void **)&ring->ring);
361302 
362
-	amdgpu_debugfs_ring_fini(ring);
363
-
364303 	dma_fence_put(ring->vmid_wait);
365304 	ring->vmid_wait = NULL;
366305 	ring->me = 0;
367306 
368307 	ring->adev->rings[ring->idx] = NULL;
369
-}
370
-
371
-static void amdgpu_ring_lru_touch_locked(struct amdgpu_device *adev,
372
-					 struct amdgpu_ring *ring)
373
-{
374
-	/* list_move_tail handles the case where ring isn't part of the list */
375
-	list_move_tail(&ring->lru_list, &adev->ring_lru_list);
376
-}
377
-
378
-static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
379
-				       int *blacklist, int num_blacklist)
380
-{
381
-	int i;
382
-
383
-	for (i = 0; i < num_blacklist; i++) {
384
-		if (ring->idx == blacklist[i])
385
-			return true;
386
-	}
387
-
388
-	return false;
389
-}
390
-
391
-/**
392
- * amdgpu_ring_lru_get - get the least recently used ring for a HW IP block
393
- *
394
- * @adev: amdgpu_device pointer
395
- * @type: amdgpu_ring_type enum
396
- * @blacklist: blacklisted ring ids array
397
- * @num_blacklist: number of entries in @blacklist
398
- * @lru_pipe_order: find a ring from the least recently used pipe
399
- * @ring: output ring
400
- *
401
- * Retrieve the amdgpu_ring structure for the least recently used ring of
402
- * a specific IP block (all asics).
403
- * Returns 0 on success, error on failure.
404
- */
405
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
406
-			int *blacklist,	int num_blacklist,
407
-			bool lru_pipe_order, struct amdgpu_ring **ring)
408
-{
409
-	struct amdgpu_ring *entry;
410
-
411
-	/* List is sorted in LRU order, find first entry corresponding
412
-	 * to the desired HW IP */
413
-	*ring = NULL;
414
-	spin_lock(&adev->ring_lru_list_lock);
415
-	list_for_each_entry(entry, &adev->ring_lru_list, lru_list) {
416
-		if (entry->funcs->type != type)
417
-			continue;
418
-
419
-		if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
420
-			continue;
421
-
422
-		if (!*ring) {
423
-			*ring = entry;
424
-
425
-			/* We are done for ring LRU */
426
-			if (!lru_pipe_order)
427
-				break;
428
-		}
429
-
430
-		/* Move all rings on the same pipe to the end of the list */
431
-		if (entry->pipe == (*ring)->pipe)
432
-			amdgpu_ring_lru_touch_locked(adev, entry);
433
-	}
434
-
435
-	/* Move the ring we found to the end of the list */
436
-	if (*ring)
437
-		amdgpu_ring_lru_touch_locked(adev, *ring);
438
-
439
-	spin_unlock(&adev->ring_lru_list_lock);
440
-
441
-	if (!*ring) {
442
-		DRM_ERROR("Ring LRU contains no entries for ring type:%d\n", type);
443
-		return -EINVAL;
444
-	}
445
-
446
-	return 0;
447
-}
448
-
449
-/**
450
- * amdgpu_ring_lru_touch - mark a ring as recently being used
451
- *
452
- * @adev: amdgpu_device pointer
453
- * @ring: ring to touch
454
- *
455
- * Move @ring to the tail of the lru list
456
- */
457
-void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring)
458
-{
459
-	spin_lock(&adev->ring_lru_list_lock);
460
-	amdgpu_ring_lru_touch_locked(adev, ring);
461
-	spin_unlock(&adev->ring_lru_list_lock);
462308 }
463309 
464310 /**
....@@ -479,6 +325,31 @@
479325 {
480326 	amdgpu_ring_emit_wreg(ring, reg0, ref);
481327 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
328
+}
329
+
330
+/**
331
+ * amdgpu_ring_soft_recovery - try to soft recover a ring lockup
332
+ *
333
+ * @ring: ring to try the recovery on
334
+ * @vmid: VMID we try to get going again
335
+ * @fence: timedout fence
336
+ *
337
+ * Tries to get a ring proceeding again when it is stuck.
338
+ */
339
+bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
340
+			       struct dma_fence *fence)
341
+{
342
+	ktime_t deadline = ktime_add_us(ktime_get(), 10000);
343
+
344
+	if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
345
+		return false;
346
+
347
+	atomic_inc(&ring->adev->gpu_reset_counter);
348
+	while (!dma_fence_is_signaled(fence) &&
349
+	       ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0)
350
+		ring->funcs->soft_recovery(ring, vmid);
351
+
352
+	return dma_fence_is_signaled(fence);
482353 }
483354 
484355 /*
....@@ -545,11 +416,11 @@
545416 
546417 #endif
547418 
548
-static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
549
-				    struct amdgpu_ring *ring)
419
+int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
420
+			     struct amdgpu_ring *ring)
550421 {
551422 #if defined(CONFIG_DEBUG_FS)
552
-	struct drm_minor *minor = adev->ddev->primary;
423
+	struct drm_minor *minor = adev_to_drm(adev)->primary;
553424 	struct dentry *ent, *root = minor->debugfs_root;
554425 	char name[32];
555426 
....@@ -567,9 +438,28 @@
567438 	return 0;
568439 }
569440 
570
-static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring)
441
+/**
442
+ * amdgpu_ring_test_helper - tests ring and set sched readiness status
443
+ *
444
+ * @ring: ring to try the recovery on
445
+ *
446
+ * Tests ring and set sched readiness status
447
+ *
448
+ * Returns 0 on success, error on failure.
449
+ */
450
+int amdgpu_ring_test_helper(struct amdgpu_ring *ring)
571451 {
572
-#if defined(CONFIG_DEBUG_FS)
573
-	debugfs_remove(ring->ent);
574
-#endif
452
+	struct amdgpu_device *adev = ring->adev;
453
+	int r;
454
+
455
+	r = amdgpu_ring_test_ring(ring);
456
+	if (r)
457
+		DRM_DEV_ERROR(adev->dev, "ring %s test failed (%d)\n",
458
+			      ring->name, r);
459
+	else
460
+		DRM_DEV_DEBUG(adev->dev, "ring test on %s succeeded\n",
461
+			      ring->name);
462
+
463
+	ring->sched.ready = !r;
464
+	return r;
575465 }