hc
2024-05-10 9999e48639b3cecb08ffb37358bcba3b48161b29
kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
....@@ -28,8 +28,9 @@
2828 */
2929 #include <linux/seq_file.h>
3030 #include <linux/slab.h>
31
+#include <linux/uaccess.h>
3132 #include <linux/debugfs.h>
32
-#include <drm/drmP.h>
33
+
3334 #include <drm/amdgpu_drm.h>
3435 #include "amdgpu.h"
3536 #include "atom.h"
....@@ -47,9 +48,6 @@
4748 * wptr. The GPU then starts fetching commands and executes
4849 * them until the pointers are equal again.
4950 */
50
-static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
51
- struct amdgpu_ring *ring);
52
-static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring);
5351
5452 /**
5553 * amdgpu_ring_alloc - allocate space on the ring buffer
....@@ -135,9 +133,6 @@
135133
136134 if (ring->funcs->end_use)
137135 ring->funcs->end_use(ring);
138
-
139
- if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ)
140
- amdgpu_ring_lru_touch(ring->adev, ring);
141136 }
142137
143138 /**
....@@ -156,76 +151,6 @@
156151 }
157152
158153 /**
159
- * amdgpu_ring_priority_put - restore a ring's priority
160
- *
161
- * @ring: amdgpu_ring structure holding the information
162
- * @priority: target priority
163
- *
164
- * Release a request for executing at @priority
165
- */
166
-void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
167
- enum drm_sched_priority priority)
168
-{
169
- int i;
170
-
171
- if (!ring->funcs->set_priority)
172
- return;
173
-
174
- if (atomic_dec_return(&ring->num_jobs[priority]) > 0)
175
- return;
176
-
177
- /* no need to restore if the job is already at the lowest priority */
178
- if (priority == DRM_SCHED_PRIORITY_NORMAL)
179
- return;
180
-
181
- mutex_lock(&ring->priority_mutex);
182
- /* something higher prio is executing, no need to decay */
183
- if (ring->priority > priority)
184
- goto out_unlock;
185
-
186
- /* decay priority to the next level with a job available */
187
- for (i = priority; i >= DRM_SCHED_PRIORITY_MIN; i--) {
188
- if (i == DRM_SCHED_PRIORITY_NORMAL
189
- || atomic_read(&ring->num_jobs[i])) {
190
- ring->priority = i;
191
- ring->funcs->set_priority(ring, i);
192
- break;
193
- }
194
- }
195
-
196
-out_unlock:
197
- mutex_unlock(&ring->priority_mutex);
198
-}
199
-
200
-/**
201
- * amdgpu_ring_priority_get - change the ring's priority
202
- *
203
- * @ring: amdgpu_ring structure holding the information
204
- * @priority: target priority
205
- *
206
- * Request a ring's priority to be raised to @priority (refcounted).
207
- */
208
-void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
209
- enum drm_sched_priority priority)
210
-{
211
- if (!ring->funcs->set_priority)
212
- return;
213
-
214
- if (atomic_inc_return(&ring->num_jobs[priority]) <= 0)
215
- return;
216
-
217
- mutex_lock(&ring->priority_mutex);
218
- if (priority <= ring->priority)
219
- goto out_unlock;
220
-
221
- ring->priority = priority;
222
- ring->funcs->set_priority(ring, priority);
223
-
224
-out_unlock:
225
- mutex_unlock(&ring->priority_mutex);
226
-}
227
-
228
-/**
229154 * amdgpu_ring_init - init driver ring struct.
230155 *
231156 * @adev: amdgpu_device pointer
....@@ -237,11 +162,13 @@
237162 * Returns 0 on success, error on failure.
238163 */
239164 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
240
- unsigned max_dw, struct amdgpu_irq_src *irq_src,
241
- unsigned irq_type)
165
+ unsigned int max_dw, struct amdgpu_irq_src *irq_src,
166
+ unsigned int irq_type, unsigned int hw_prio)
242167 {
243168 int r, i;
244169 int sched_hw_submission = amdgpu_sched_hw_submission;
170
+ u32 *num_sched;
171
+ u32 hw_ip;
245172
246173 /* Set the hw submission limit higher for KIQ because
247174 * it's used for a number of gfx/compute tasks by both
....@@ -251,6 +178,8 @@
251178 */
252179 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
253180 sched_hw_submission = max(sched_hw_submission, 256);
181
+ else if (ring == &adev->sdma.instance[0].page)
182
+ sched_hw_submission = 256;
254183
255184 if (ring->adev == NULL) {
256185 if (adev->num_rings >= AMDGPU_MAX_RINGS)
....@@ -281,6 +210,16 @@
281210 dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
282211 return r;
283212 }
213
+
214
+ r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs);
215
+ if (r) {
216
+ dev_err(adev->dev,
217
+ "(%d) ring trail_fence_offs wb alloc failed\n", r);
218
+ return r;
219
+ }
220
+ ring->trail_fence_gpu_addr =
221
+ adev->wb.gpu_addr + (ring->trail_fence_offs * 4);
222
+ ring->trail_fence_cpu_addr = &adev->wb.wb[ring->trail_fence_offs];
284223
285224 r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);
286225 if (r) {
....@@ -320,15 +259,16 @@
320259 ring->max_dw = max_dw;
321260 ring->priority = DRM_SCHED_PRIORITY_NORMAL;
322261 mutex_init(&ring->priority_mutex);
323
- INIT_LIST_HEAD(&ring->lru_list);
324
- amdgpu_ring_lru_touch(adev, ring);
325262
326
- for (i = 0; i < DRM_SCHED_PRIORITY_MAX; ++i)
327
- atomic_set(&ring->num_jobs[i], 0);
328
-
329
- if (amdgpu_debugfs_ring_init(adev, ring)) {
330
- DRM_ERROR("Failed to register debugfs file for rings !\n");
263
+ if (!ring->no_scheduler) {
264
+ hw_ip = ring->funcs->type;
265
+ num_sched = &adev->gpu_sched[hw_ip][hw_prio].num_scheds;
266
+ adev->gpu_sched[hw_ip][hw_prio].sched[(*num_sched)++] =
267
+ &ring->sched;
331268 }
269
+
270
+ for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; ++i)
271
+ atomic_set(&ring->num_jobs[i], 0);
332272
333273 return 0;
334274 }
....@@ -343,11 +283,12 @@
343283 */
344284 void amdgpu_ring_fini(struct amdgpu_ring *ring)
345285 {
346
- ring->ready = false;
347286
348287 /* Not to finish a ring which is not initialized */
349288 if (!(ring->adev) || !(ring->adev->rings[ring->idx]))
350289 return;
290
+
291
+ ring->sched.ready = false;
351292
352293 amdgpu_device_wb_free(ring->adev, ring->rptr_offs);
353294 amdgpu_device_wb_free(ring->adev, ring->wptr_offs);
....@@ -359,106 +300,11 @@
359300 &ring->gpu_addr,
360301 (void **)&ring->ring);
361302
362
- amdgpu_debugfs_ring_fini(ring);
363
-
364303 dma_fence_put(ring->vmid_wait);
365304 ring->vmid_wait = NULL;
366305 ring->me = 0;
367306
368307 ring->adev->rings[ring->idx] = NULL;
369
-}
370
-
371
-static void amdgpu_ring_lru_touch_locked(struct amdgpu_device *adev,
372
- struct amdgpu_ring *ring)
373
-{
374
- /* list_move_tail handles the case where ring isn't part of the list */
375
- list_move_tail(&ring->lru_list, &adev->ring_lru_list);
376
-}
377
-
378
-static bool amdgpu_ring_is_blacklisted(struct amdgpu_ring *ring,
379
- int *blacklist, int num_blacklist)
380
-{
381
- int i;
382
-
383
- for (i = 0; i < num_blacklist; i++) {
384
- if (ring->idx == blacklist[i])
385
- return true;
386
- }
387
-
388
- return false;
389
-}
390
-
391
-/**
392
- * amdgpu_ring_lru_get - get the least recently used ring for a HW IP block
393
- *
394
- * @adev: amdgpu_device pointer
395
- * @type: amdgpu_ring_type enum
396
- * @blacklist: blacklisted ring ids array
397
- * @num_blacklist: number of entries in @blacklist
398
- * @lru_pipe_order: find a ring from the least recently used pipe
399
- * @ring: output ring
400
- *
401
- * Retrieve the amdgpu_ring structure for the least recently used ring of
402
- * a specific IP block (all asics).
403
- * Returns 0 on success, error on failure.
404
- */
405
-int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
406
- int *blacklist, int num_blacklist,
407
- bool lru_pipe_order, struct amdgpu_ring **ring)
408
-{
409
- struct amdgpu_ring *entry;
410
-
411
- /* List is sorted in LRU order, find first entry corresponding
412
- * to the desired HW IP */
413
- *ring = NULL;
414
- spin_lock(&adev->ring_lru_list_lock);
415
- list_for_each_entry(entry, &adev->ring_lru_list, lru_list) {
416
- if (entry->funcs->type != type)
417
- continue;
418
-
419
- if (amdgpu_ring_is_blacklisted(entry, blacklist, num_blacklist))
420
- continue;
421
-
422
- if (!*ring) {
423
- *ring = entry;
424
-
425
- /* We are done for ring LRU */
426
- if (!lru_pipe_order)
427
- break;
428
- }
429
-
430
- /* Move all rings on the same pipe to the end of the list */
431
- if (entry->pipe == (*ring)->pipe)
432
- amdgpu_ring_lru_touch_locked(adev, entry);
433
- }
434
-
435
- /* Move the ring we found to the end of the list */
436
- if (*ring)
437
- amdgpu_ring_lru_touch_locked(adev, *ring);
438
-
439
- spin_unlock(&adev->ring_lru_list_lock);
440
-
441
- if (!*ring) {
442
- DRM_ERROR("Ring LRU contains no entries for ring type:%d\n", type);
443
- return -EINVAL;
444
- }
445
-
446
- return 0;
447
-}
448
-
449
-/**
450
- * amdgpu_ring_lru_touch - mark a ring as recently being used
451
- *
452
- * @adev: amdgpu_device pointer
453
- * @ring: ring to touch
454
- *
455
- * Move @ring to the tail of the lru list
456
- */
457
-void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring)
458
-{
459
- spin_lock(&adev->ring_lru_list_lock);
460
- amdgpu_ring_lru_touch_locked(adev, ring);
461
- spin_unlock(&adev->ring_lru_list_lock);
462308 }
463309
464310 /**
....@@ -479,6 +325,31 @@
479325 {
480326 amdgpu_ring_emit_wreg(ring, reg0, ref);
481327 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
328
+}
329
+
330
+/**
331
+ * amdgpu_ring_soft_recovery - try to soft recover a ring lockup
332
+ *
333
+ * @ring: ring to try the recovery on
334
+ * @vmid: VMID we try to get going again
335
+ * @fence: timedout fence
336
+ *
337
+ * Tries to get a ring proceeding again when it is stuck.
338
+ */
339
+bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
340
+ struct dma_fence *fence)
341
+{
342
+ ktime_t deadline = ktime_add_us(ktime_get(), 10000);
343
+
344
+ if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
345
+ return false;
346
+
347
+ atomic_inc(&ring->adev->gpu_reset_counter);
348
+ while (!dma_fence_is_signaled(fence) &&
349
+ ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0)
350
+ ring->funcs->soft_recovery(ring, vmid);
351
+
352
+ return dma_fence_is_signaled(fence);
482353 }
483354
484355 /*
....@@ -545,11 +416,11 @@
545416
546417 #endif
547418
548
-static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
549
- struct amdgpu_ring *ring)
419
+int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
420
+ struct amdgpu_ring *ring)
550421 {
551422 #if defined(CONFIG_DEBUG_FS)
552
- struct drm_minor *minor = adev->ddev->primary;
423
+ struct drm_minor *minor = adev_to_drm(adev)->primary;
553424 struct dentry *ent, *root = minor->debugfs_root;
554425 char name[32];
555426
....@@ -567,9 +438,28 @@
567438 return 0;
568439 }
569440
570
-static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring)
441
+/**
442
+ * amdgpu_ring_test_helper - tests ring and set sched readiness status
443
+ *
444
+ * @ring: ring to try the recovery on
445
+ *
446
+ * Tests ring and set sched readiness status
447
+ *
448
+ * Returns 0 on success, error on failure.
449
+ */
450
+int amdgpu_ring_test_helper(struct amdgpu_ring *ring)
571451 {
572
-#if defined(CONFIG_DEBUG_FS)
573
- debugfs_remove(ring->ent);
574
-#endif
452
+ struct amdgpu_device *adev = ring->adev;
453
+ int r;
454
+
455
+ r = amdgpu_ring_test_ring(ring);
456
+ if (r)
457
+ DRM_DEV_ERROR(adev->dev, "ring %s test failed (%d)\n",
458
+ ring->name, r);
459
+ else
460
+ DRM_DEV_DEBUG(adev->dev, "ring test on %s succeeded\n",
461
+ ring->name);
462
+
463
+ ring->sched.ready = !r;
464
+ return r;
575465 }