| .. | .. |
|---|
| 34 | 34 | #include <linux/kref.h> |
|---|
| 35 | 35 | #include <linux/slab.h> |
|---|
| 36 | 36 | #include <linux/firmware.h> |
|---|
| 37 | | -#include <drm/drmP.h> |
|---|
| 37 | +#include <linux/pm_runtime.h> |
|---|
| 38 | + |
|---|
| 39 | +#include <drm/drm_debugfs.h> |
|---|
| 40 | + |
|---|
| 38 | 41 | #include "amdgpu.h" |
|---|
| 39 | 42 | #include "amdgpu_trace.h" |
|---|
| 40 | 43 | |
|---|
| .. | .. |
|---|
| 152 | 155 | seq); |
|---|
| 153 | 156 | amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, |
|---|
| 154 | 157 | seq, flags | AMDGPU_FENCE_FLAG_INT); |
|---|
| 155 | | - |
|---|
| 158 | + pm_runtime_get_noresume(adev_to_drm(adev)->dev); |
|---|
| 156 | 159 | ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask]; |
|---|
| 157 | 160 | if (unlikely(rcu_dereference_protected(*ptr, 1))) { |
|---|
| 158 | 161 | struct dma_fence *old; |
|---|
| .. | .. |
|---|
| 189 | 192 | * Used For polling fence. |
|---|
| 190 | 193 | * Returns 0 on success, -ENOMEM on failure. |
|---|
| 191 | 194 | */ |
|---|
| 192 | | -int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s) |
|---|
| 195 | +int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s, |
|---|
| 196 | + uint32_t timeout) |
|---|
| 193 | 197 | { |
|---|
| 194 | 198 | uint32_t seq; |
|---|
| 199 | + signed long r; |
|---|
| 195 | 200 | |
|---|
| 196 | 201 | if (!s) |
|---|
| 197 | 202 | return -EINVAL; |
|---|
| 198 | 203 | |
|---|
| 199 | 204 | seq = ++ring->fence_drv.sync_seq; |
|---|
| 205 | + r = amdgpu_fence_wait_polling(ring, |
|---|
| 206 | + seq - ring->fence_drv.num_fences_mask, |
|---|
| 207 | + timeout); |
|---|
| 208 | + if (r < 1) |
|---|
| 209 | + return -ETIMEDOUT; |
|---|
| 210 | + |
|---|
| 200 | 211 | amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, |
|---|
| 201 | 212 | seq, 0); |
|---|
| 202 | 213 | |
|---|
| .. | .. |
|---|
| 226 | 237 | * Checks the current fence value and calculates the last |
|---|
| 227 | 238 | * signalled fence value. Wakes the fence queue if the |
|---|
| 228 | 239 | * sequence number has increased. |
|---|
| 240 | + * |
|---|
| 241 | + * Returns true if fence was processed |
|---|
| 229 | 242 | */ |
|---|
| 230 | | -void amdgpu_fence_process(struct amdgpu_ring *ring) |
|---|
| 243 | +bool amdgpu_fence_process(struct amdgpu_ring *ring) |
|---|
| 231 | 244 | { |
|---|
| 232 | 245 | struct amdgpu_fence_driver *drv = &ring->fence_drv; |
|---|
| 246 | + struct amdgpu_device *adev = ring->adev; |
|---|
| 233 | 247 | uint32_t seq, last_seq; |
|---|
| 234 | 248 | int r; |
|---|
| 235 | 249 | |
|---|
| .. | .. |
|---|
| 239 | 253 | |
|---|
| 240 | 254 | } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq); |
|---|
| 241 | 255 | |
|---|
| 242 | | - if (seq != ring->fence_drv.sync_seq) |
|---|
| 256 | + if (del_timer(&ring->fence_drv.fallback_timer) && |
|---|
| 257 | + seq != ring->fence_drv.sync_seq) |
|---|
| 243 | 258 | amdgpu_fence_schedule_fallback(ring); |
|---|
| 244 | 259 | |
|---|
| 245 | 260 | if (unlikely(seq == last_seq)) |
|---|
| 246 | | - return; |
|---|
| 261 | + return false; |
|---|
| 247 | 262 | |
|---|
| 248 | 263 | last_seq &= drv->num_fences_mask; |
|---|
| 249 | 264 | seq &= drv->num_fences_mask; |
|---|
| .. | .. |
|---|
| 269 | 284 | BUG(); |
|---|
| 270 | 285 | |
|---|
| 271 | 286 | dma_fence_put(fence); |
|---|
| 287 | + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); |
|---|
| 288 | + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); |
|---|
| 272 | 289 | } while (last_seq != seq); |
|---|
| 290 | + |
|---|
| 291 | + return true; |
|---|
| 273 | 292 | } |
|---|
| 274 | 293 | |
|---|
| 275 | 294 | /** |
|---|
| .. | .. |
|---|
| 284 | 303 | struct amdgpu_ring *ring = from_timer(ring, t, |
|---|
| 285 | 304 | fence_drv.fallback_timer); |
|---|
| 286 | 305 | |
|---|
| 287 | | - amdgpu_fence_process(ring); |
|---|
| 306 | + if (amdgpu_fence_process(ring)) |
|---|
| 307 | + DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name); |
|---|
| 288 | 308 | } |
|---|
| 289 | 309 | |
|---|
| 290 | 310 | /** |
|---|
| .. | .. |
|---|
| 404 | 424 | ring->fence_drv.irq_type = irq_type; |
|---|
| 405 | 425 | ring->fence_drv.initialized = true; |
|---|
| 406 | 426 | |
|---|
| 407 | | - dev_dbg(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, " |
|---|
| 408 | | - "cpu addr 0x%p\n", ring->idx, |
|---|
| 409 | | - ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr); |
|---|
| 427 | + DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n", |
|---|
| 428 | + ring->name, ring->fence_drv.gpu_addr); |
|---|
| 410 | 429 | return 0; |
|---|
| 411 | 430 | } |
|---|
| 412 | 431 | |
|---|
| .. | .. |
|---|
| 423 | 442 | int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring, |
|---|
| 424 | 443 | unsigned num_hw_submission) |
|---|
| 425 | 444 | { |
|---|
| 445 | + struct amdgpu_device *adev = ring->adev; |
|---|
| 426 | 446 | long timeout; |
|---|
| 427 | 447 | int r; |
|---|
| 428 | 448 | |
|---|
| 429 | | - /* Check that num_hw_submission is a power of two */ |
|---|
| 430 | | - if ((num_hw_submission & (num_hw_submission - 1)) != 0) |
|---|
| 449 | + if (!adev) |
|---|
| 450 | + return -EINVAL; |
|---|
| 451 | + |
|---|
| 452 | + if (!is_power_of_2(num_hw_submission)) |
|---|
| 431 | 453 | return -EINVAL; |
|---|
| 432 | 454 | |
|---|
| 433 | 455 | ring->fence_drv.cpu_addr = NULL; |
|---|
| .. | .. |
|---|
| 445 | 467 | if (!ring->fence_drv.fences) |
|---|
| 446 | 468 | return -ENOMEM; |
|---|
| 447 | 469 | |
|---|
| 448 | | - /* No need to setup the GPU scheduler for KIQ ring */ |
|---|
| 449 | | - if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) { |
|---|
| 450 | | - /* for non-sriov case, no timeout enforce on compute ring */ |
|---|
| 451 | | - if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) |
|---|
| 452 | | - && !amdgpu_sriov_vf(ring->adev)) |
|---|
| 453 | | - timeout = MAX_SCHEDULE_TIMEOUT; |
|---|
| 454 | | - else |
|---|
| 455 | | - timeout = msecs_to_jiffies(amdgpu_lockup_timeout); |
|---|
| 470 | + /* No need to setup the GPU scheduler for rings that don't need it */ |
|---|
| 471 | + if (!ring->no_scheduler) { |
|---|
| 472 | + switch (ring->funcs->type) { |
|---|
| 473 | + case AMDGPU_RING_TYPE_GFX: |
|---|
| 474 | + timeout = adev->gfx_timeout; |
|---|
| 475 | + break; |
|---|
| 476 | + case AMDGPU_RING_TYPE_COMPUTE: |
|---|
| 477 | + timeout = adev->compute_timeout; |
|---|
| 478 | + break; |
|---|
| 479 | + case AMDGPU_RING_TYPE_SDMA: |
|---|
| 480 | + timeout = adev->sdma_timeout; |
|---|
| 481 | + break; |
|---|
| 482 | + default: |
|---|
| 483 | + timeout = adev->video_timeout; |
|---|
| 484 | + break; |
|---|
| 485 | + } |
|---|
| 456 | 486 | |
|---|
| 457 | 487 | r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, |
|---|
| 458 | 488 | num_hw_submission, amdgpu_job_hang_limit, |
|---|
| .. | .. |
|---|
| 481 | 511 | */ |
|---|
| 482 | 512 | int amdgpu_fence_driver_init(struct amdgpu_device *adev) |
|---|
| 483 | 513 | { |
|---|
| 484 | | - if (amdgpu_debugfs_fence_init(adev)) |
|---|
| 485 | | - dev_err(adev->dev, "fence debugfs file creation failed\n"); |
|---|
| 486 | | - |
|---|
| 487 | 514 | return 0; |
|---|
| 488 | 515 | } |
|---|
| 489 | 516 | |
|---|
| .. | .. |
|---|
| 505 | 532 | |
|---|
| 506 | 533 | if (!ring || !ring->fence_drv.initialized) |
|---|
| 507 | 534 | continue; |
|---|
| 535 | + if (!ring->no_scheduler) |
|---|
| 536 | + drm_sched_fini(&ring->sched); |
|---|
| 508 | 537 | r = amdgpu_fence_wait_empty(ring); |
|---|
| 509 | 538 | if (r) { |
|---|
| 510 | 539 | /* no need to trigger GPU reset as we are unloading */ |
|---|
| .. | .. |
|---|
| 513 | 542 | if (ring->fence_drv.irq_src) |
|---|
| 514 | 543 | amdgpu_irq_put(adev, ring->fence_drv.irq_src, |
|---|
| 515 | 544 | ring->fence_drv.irq_type); |
|---|
| 516 | | - drm_sched_fini(&ring->sched); |
|---|
| 545 | + |
|---|
| 517 | 546 | del_timer_sync(&ring->fence_drv.fallback_timer); |
|---|
| 518 | 547 | for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j) |
|---|
| 519 | 548 | dma_fence_put(ring->fence_drv.fences[j]); |
|---|
| .. | .. |
|---|
| 672 | 701 | { |
|---|
| 673 | 702 | struct drm_info_node *node = (struct drm_info_node *)m->private; |
|---|
| 674 | 703 | struct drm_device *dev = node->minor->dev; |
|---|
| 675 | | - struct amdgpu_device *adev = dev->dev_private; |
|---|
| 704 | + struct amdgpu_device *adev = drm_to_adev(dev); |
|---|
| 676 | 705 | int i; |
|---|
| 677 | 706 | |
|---|
| 678 | 707 | for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { |
|---|
| .. | .. |
|---|
| 683 | 712 | amdgpu_fence_process(ring); |
|---|
| 684 | 713 | |
|---|
| 685 | 714 | seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name); |
|---|
| 686 | | - seq_printf(m, "Last signaled fence 0x%08x\n", |
|---|
| 715 | + seq_printf(m, "Last signaled fence 0x%08x\n", |
|---|
| 687 | 716 | atomic_read(&ring->fence_drv.last_seq)); |
|---|
| 688 | | - seq_printf(m, "Last emitted 0x%08x\n", |
|---|
| 717 | + seq_printf(m, "Last emitted 0x%08x\n", |
|---|
| 689 | 718 | ring->fence_drv.sync_seq); |
|---|
| 719 | + |
|---|
| 720 | + if (ring->funcs->type == AMDGPU_RING_TYPE_GFX || |
|---|
| 721 | + ring->funcs->type == AMDGPU_RING_TYPE_SDMA) { |
|---|
| 722 | + seq_printf(m, "Last signaled trailing fence 0x%08x\n", |
|---|
| 723 | + le32_to_cpu(*ring->trail_fence_cpu_addr)); |
|---|
| 724 | + seq_printf(m, "Last emitted 0x%08x\n", |
|---|
| 725 | + ring->trail_seq); |
|---|
| 726 | + } |
|---|
| 690 | 727 | |
|---|
| 691 | 728 | if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) |
|---|
| 692 | 729 | continue; |
|---|
| 693 | 730 | |
|---|
| 694 | 731 | /* set in CP_VMID_PREEMPT and preemption occurred */ |
|---|
| 695 | | - seq_printf(m, "Last preempted 0x%08x\n", |
|---|
| 732 | + seq_printf(m, "Last preempted 0x%08x\n", |
|---|
| 696 | 733 | le32_to_cpu(*(ring->fence_drv.cpu_addr + 2))); |
|---|
| 697 | 734 | /* set in CP_VMID_RESET and reset occurred */ |
|---|
| 698 | | - seq_printf(m, "Last reset 0x%08x\n", |
|---|
| 735 | + seq_printf(m, "Last reset 0x%08x\n", |
|---|
| 699 | 736 | le32_to_cpu(*(ring->fence_drv.cpu_addr + 4))); |
|---|
| 700 | 737 | /* Both preemption and reset occurred */ |
|---|
| 701 | | - seq_printf(m, "Last both 0x%08x\n", |
|---|
| 738 | + seq_printf(m, "Last both 0x%08x\n", |
|---|
| 702 | 739 | le32_to_cpu(*(ring->fence_drv.cpu_addr + 6))); |
|---|
| 703 | 740 | } |
|---|
| 704 | 741 | return 0; |
|---|
| .. | .. |
|---|
| 713 | 750 | { |
|---|
| 714 | 751 | struct drm_info_node *node = (struct drm_info_node *) m->private; |
|---|
| 715 | 752 | struct drm_device *dev = node->minor->dev; |
|---|
| 716 | | - struct amdgpu_device *adev = dev->dev_private; |
|---|
| 753 | + struct amdgpu_device *adev = drm_to_adev(dev); |
|---|
| 754 | + int r; |
|---|
| 755 | + |
|---|
| 756 | + r = pm_runtime_get_sync(dev->dev); |
|---|
| 757 | + if (r < 0) { |
|---|
| 758 | + pm_runtime_put_autosuspend(dev->dev); |
|---|
| 759 | + return 0; |
|---|
| 760 | + } |
|---|
| 717 | 761 | |
|---|
| 718 | 762 | seq_printf(m, "gpu recover\n"); |
|---|
| 719 | | - amdgpu_device_gpu_recover(adev, NULL, true); |
|---|
| 763 | + amdgpu_device_gpu_recover(adev, NULL); |
|---|
| 764 | + |
|---|
| 765 | + pm_runtime_mark_last_busy(dev->dev); |
|---|
| 766 | + pm_runtime_put_autosuspend(dev->dev); |
|---|
| 720 | 767 | |
|---|
| 721 | 768 | return 0; |
|---|
| 722 | 769 | } |
|---|
| .. | .. |
|---|
| 735 | 782 | { |
|---|
| 736 | 783 | #if defined(CONFIG_DEBUG_FS) |
|---|
| 737 | 784 | if (amdgpu_sriov_vf(adev)) |
|---|
| 738 | | - return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list_sriov, 1); |
|---|
| 739 | | - return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, 2); |
|---|
| 785 | + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list_sriov, |
|---|
| 786 | + ARRAY_SIZE(amdgpu_debugfs_fence_list_sriov)); |
|---|
| 787 | + return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_fence_list, |
|---|
| 788 | + ARRAY_SIZE(amdgpu_debugfs_fence_list)); |
|---|
| 740 | 789 | #else |
|---|
| 741 | 790 | return 0; |
|---|
| 742 | 791 | #endif |
|---|