| .. | .. |
|---|
| 24 | 24 | |
|---|
| 25 | 25 | #include <linux/idr.h> |
|---|
| 26 | 26 | #include <linux/dma-fence-array.h> |
|---|
| 27 | | -#include <drm/drmP.h> |
|---|
| 27 | + |
|---|
| 28 | 28 | |
|---|
| 29 | 29 | #include "amdgpu.h" |
|---|
| 30 | 30 | #include "amdgpu_trace.h" |
|---|
| .. | .. |
|---|
| 43 | 43 | /* Helper to free pasid from a fence callback */ |
|---|
| 44 | 44 | struct amdgpu_pasid_cb { |
|---|
| 45 | 45 | struct dma_fence_cb cb; |
|---|
| 46 | | - unsigned int pasid; |
|---|
| 46 | + u32 pasid; |
|---|
| 47 | 47 | }; |
|---|
| 48 | 48 | |
|---|
| 49 | 49 | /** |
|---|
| .. | .. |
|---|
| 79 | 79 | * amdgpu_pasid_free - Free a PASID |
|---|
| 80 | 80 | * @pasid: PASID to free |
|---|
| 81 | 81 | */ |
|---|
| 82 | | -void amdgpu_pasid_free(unsigned int pasid) |
|---|
| 82 | +void amdgpu_pasid_free(u32 pasid) |
|---|
| 83 | 83 | { |
|---|
| 84 | 84 | trace_amdgpu_pasid_freed(pasid); |
|---|
| 85 | 85 | ida_simple_remove(&amdgpu_pasid_ida, pasid); |
|---|
| .. | .. |
|---|
| 104 | 104 | * |
|---|
| 105 | 105 | * Free the pasid only after all the fences in resv are signaled. |
|---|
| 106 | 106 | */ |
|---|
| 107 | | -void amdgpu_pasid_free_delayed(struct reservation_object *resv, |
|---|
| 108 | | - unsigned int pasid) |
|---|
| 107 | +void amdgpu_pasid_free_delayed(struct dma_resv *resv, |
|---|
| 108 | + u32 pasid) |
|---|
| 109 | 109 | { |
|---|
| 110 | 110 | struct dma_fence *fence, **fences; |
|---|
| 111 | 111 | struct amdgpu_pasid_cb *cb; |
|---|
| 112 | 112 | unsigned count; |
|---|
| 113 | 113 | int r; |
|---|
| 114 | 114 | |
|---|
| 115 | | - r = reservation_object_get_fences_rcu(resv, NULL, &count, &fences); |
|---|
| 115 | + r = dma_resv_get_fences_rcu(resv, NULL, &count, &fences); |
|---|
| 116 | 116 | if (r) |
|---|
| 117 | 117 | goto fallback; |
|---|
| 118 | 118 | |
|---|
| .. | .. |
|---|
| 156 | 156 | /* Not enough memory for the delayed delete, as last resort |
|---|
| 157 | 157 | * block for all the fences to complete. |
|---|
| 158 | 158 | */ |
|---|
| 159 | | - reservation_object_wait_timeout_rcu(resv, true, false, |
|---|
| 159 | + dma_resv_wait_timeout_rcu(resv, true, false, |
|---|
| 160 | 160 | MAX_SCHEDULE_TIMEOUT); |
|---|
| 161 | 161 | amdgpu_pasid_free(pasid); |
|---|
| 162 | 162 | } |
|---|
| .. | .. |
|---|
| 206 | 206 | int r; |
|---|
| 207 | 207 | |
|---|
| 208 | 208 | if (ring->vmid_wait && !dma_fence_is_signaled(ring->vmid_wait)) |
|---|
| 209 | | - return amdgpu_sync_fence(adev, sync, ring->vmid_wait, false); |
|---|
| 209 | + return amdgpu_sync_fence(sync, ring->vmid_wait); |
|---|
| 210 | 210 | |
|---|
| 211 | 211 | fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL); |
|---|
| 212 | 212 | if (!fences) |
|---|
| .. | .. |
|---|
| 215 | 215 | /* Check if we have an idle VMID */ |
|---|
| 216 | 216 | i = 0; |
|---|
| 217 | 217 | list_for_each_entry((*idle), &id_mgr->ids_lru, list) { |
|---|
| 218 | | - fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, ring); |
|---|
| 218 | + /* Don't use per engine and per process VMID at the same time */ |
|---|
| 219 | + struct amdgpu_ring *r = adev->vm_manager.concurrent_flush ? |
|---|
| 220 | + NULL : ring; |
|---|
| 221 | + |
|---|
| 222 | + fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, r); |
|---|
| 219 | 223 | if (!fences[i]) |
|---|
| 220 | 224 | break; |
|---|
| 221 | 225 | ++i; |
|---|
| .. | .. |
|---|
| 241 | 245 | return -ENOMEM; |
|---|
| 242 | 246 | } |
|---|
| 243 | 247 | |
|---|
| 244 | | - r = amdgpu_sync_fence(adev, sync, &array->base, false); |
|---|
| 248 | + r = amdgpu_sync_fence(sync, &array->base); |
|---|
| 245 | 249 | dma_fence_put(ring->vmid_wait); |
|---|
| 246 | 250 | ring->vmid_wait = &array->base; |
|---|
| 247 | 251 | return r; |
|---|
| .. | .. |
|---|
| 280 | 284 | if (updates && (*id)->flushed_updates && |
|---|
| 281 | 285 | updates->context == (*id)->flushed_updates->context && |
|---|
| 282 | 286 | !dma_fence_is_later(updates, (*id)->flushed_updates)) |
|---|
| 283 | | - updates = NULL; |
|---|
| 287 | + updates = NULL; |
|---|
| 284 | 288 | |
|---|
| 285 | | - if ((*id)->owner != vm->entity.fence_context || |
|---|
| 289 | + if ((*id)->owner != vm->immediate.fence_context || |
|---|
| 286 | 290 | job->vm_pd_addr != (*id)->pd_gpu_addr || |
|---|
| 287 | 291 | updates || !(*id)->last_flush || |
|---|
| 288 | 292 | ((*id)->last_flush->context != fence_context && |
|---|
| 289 | 293 | !dma_fence_is_signaled((*id)->last_flush))) { |
|---|
| 290 | 294 | struct dma_fence *tmp; |
|---|
| 291 | 295 | |
|---|
| 296 | + /* Don't use per engine and per process VMID at the same time */ |
|---|
| 297 | + if (adev->vm_manager.concurrent_flush) |
|---|
| 298 | + ring = NULL; |
|---|
| 299 | + |
|---|
| 292 | 300 | /* to prevent one context starved by another context */ |
|---|
| 293 | 301 | (*id)->pd_gpu_addr = 0; |
|---|
| 294 | 302 | tmp = amdgpu_sync_peek_fence(&(*id)->active, ring); |
|---|
| 295 | 303 | if (tmp) { |
|---|
| 296 | 304 | *id = NULL; |
|---|
| 297 | | - r = amdgpu_sync_fence(adev, sync, tmp, false); |
|---|
| 305 | + r = amdgpu_sync_fence(sync, tmp); |
|---|
| 298 | 306 | return r; |
|---|
| 299 | 307 | } |
|---|
| 300 | 308 | needs_flush = true; |
|---|
| .. | .. |
|---|
| 303 | 311 | /* Good we can use this VMID. Remember this submission as |
|---|
| 304 | 312 | * user of the VMID. |
|---|
| 305 | 313 | */ |
|---|
| 306 | | - r = amdgpu_sync_fence(ring->adev, &(*id)->active, fence, false); |
|---|
| 314 | + r = amdgpu_sync_fence(&(*id)->active, fence); |
|---|
| 307 | 315 | if (r) |
|---|
| 308 | 316 | return r; |
|---|
| 309 | 317 | |
|---|
| .. | .. |
|---|
| 349 | 357 | struct dma_fence *flushed; |
|---|
| 350 | 358 | |
|---|
| 351 | 359 | /* Check all the prerequisites to using this VMID */ |
|---|
| 352 | | - if ((*id)->owner != vm->entity.fence_context) |
|---|
| 360 | + if ((*id)->owner != vm->immediate.fence_context) |
|---|
| 353 | 361 | continue; |
|---|
| 354 | 362 | |
|---|
| 355 | 363 | if ((*id)->pd_gpu_addr != job->vm_pd_addr) |
|---|
| .. | .. |
|---|
| 364 | 372 | if (updates && (!flushed || dma_fence_is_later(updates, flushed))) |
|---|
| 365 | 373 | needs_flush = true; |
|---|
| 366 | 374 | |
|---|
| 367 | | - /* Concurrent flushes are only possible starting with Vega10 */ |
|---|
| 368 | | - if (adev->asic_type < CHIP_VEGA10 && needs_flush) |
|---|
| 375 | + if (needs_flush && !adev->vm_manager.concurrent_flush) |
|---|
| 369 | 376 | continue; |
|---|
| 370 | 377 | |
|---|
| 371 | 378 | /* Good, we can use this VMID. Remember this submission as |
|---|
| 372 | 379 | * user of the VMID. |
|---|
| 373 | 380 | */ |
|---|
| 374 | | - r = amdgpu_sync_fence(ring->adev, &(*id)->active, fence, false); |
|---|
| 381 | + r = amdgpu_sync_fence(&(*id)->active, fence); |
|---|
| 375 | 382 | if (r) |
|---|
| 376 | 383 | return r; |
|---|
| 377 | 384 | |
|---|
| .. | .. |
|---|
| 431 | 438 | id = idle; |
|---|
| 432 | 439 | |
|---|
| 433 | 440 | /* Remember this submission as user of the VMID */ |
|---|
| 434 | | - r = amdgpu_sync_fence(ring->adev, &id->active, |
|---|
| 435 | | - fence, false); |
|---|
| 441 | + r = amdgpu_sync_fence(&id->active, fence); |
|---|
| 436 | 442 | if (r) |
|---|
| 437 | 443 | goto error; |
|---|
| 438 | 444 | |
|---|
| .. | .. |
|---|
| 445 | 451 | } |
|---|
| 446 | 452 | |
|---|
| 447 | 453 | id->pd_gpu_addr = job->vm_pd_addr; |
|---|
| 448 | | - id->owner = vm->entity.fence_context; |
|---|
| 454 | + id->owner = vm->immediate.fence_context; |
|---|
| 449 | 455 | |
|---|
| 450 | 456 | if (job->vm_needs_flush) { |
|---|
| 451 | 457 | dma_fence_put(id->last_flush); |
|---|
| .. | .. |
|---|
| 571 | 577 | INIT_LIST_HEAD(&id_mgr->ids_lru); |
|---|
| 572 | 578 | atomic_set(&id_mgr->reserved_vmid_num, 0); |
|---|
| 573 | 579 | |
|---|
| 580 | + /* manage only VMIDs not used by KFD */ |
|---|
| 581 | + id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; |
|---|
| 582 | + |
|---|
| 574 | 583 | /* skip over VMID 0, since it is the system VM */ |
|---|
| 575 | 584 | for (j = 1; j < id_mgr->num_ids; ++j) { |
|---|
| 576 | 585 | amdgpu_vmid_reset(adev, i, j); |
|---|