| .. | .. |
|---|
| 30 | 30 | |
|---|
| 31 | 31 | #include <drm/amdgpu_drm.h> |
|---|
| 32 | 32 | #include "amdgpu.h" |
|---|
| 33 | +#ifdef CONFIG_MMU_NOTIFIER |
|---|
| 34 | +#include <linux/mmu_notifier.h> |
|---|
| 35 | +#endif |
|---|
| 33 | 36 | |
|---|
| 34 | 37 | #define AMDGPU_BO_INVALID_OFFSET LONG_MAX |
|---|
| 35 | 38 | #define AMDGPU_BO_MAX_PLACEMENTS 3 |
|---|
| .. | .. |
|---|
| 41 | 44 | u32 preferred_domain; |
|---|
| 42 | 45 | u64 flags; |
|---|
| 43 | 46 | enum ttm_bo_type type; |
|---|
| 44 | | - struct reservation_object *resv; |
|---|
| 47 | + bool no_wait_gpu; |
|---|
| 48 | + struct dma_resv *resv; |
|---|
| 45 | 49 | }; |
|---|
| 46 | 50 | |
|---|
| 47 | 51 | /* bo virtual addresses in a vm */ |
|---|
| .. | .. |
|---|
| 72 | 76 | |
|---|
| 73 | 77 | /* If the mappings are cleared or filled */ |
|---|
| 74 | 78 | bool cleared; |
|---|
| 79 | + |
|---|
| 80 | + bool is_xgmi; |
|---|
| 75 | 81 | }; |
|---|
| 76 | 82 | |
|---|
| 77 | 83 | struct amdgpu_bo { |
|---|
| .. | .. |
|---|
| 89 | 95 | void *metadata; |
|---|
| 90 | 96 | u32 metadata_size; |
|---|
| 91 | 97 | unsigned prime_shared_count; |
|---|
| 92 | | - /* list of all virtual address to which this bo is associated to */ |
|---|
| 93 | | - struct list_head va; |
|---|
| 98 | + /* per VM structure for page tables and with virtual addresses */ |
|---|
| 99 | + struct amdgpu_vm_bo_base *vm_bo; |
|---|
| 94 | 100 | /* Constant after initialization */ |
|---|
| 95 | | - struct drm_gem_object gem_base; |
|---|
| 96 | 101 | struct amdgpu_bo *parent; |
|---|
| 97 | 102 | struct amdgpu_bo *shadow; |
|---|
| 98 | 103 | |
|---|
| 99 | 104 | struct ttm_bo_kmap_obj dma_buf_vmap; |
|---|
| 100 | 105 | struct amdgpu_mn *mn; |
|---|
| 101 | 106 | |
|---|
| 102 | | - union { |
|---|
| 103 | | - struct list_head mn_list; |
|---|
| 104 | | - struct list_head shadow_list; |
|---|
| 105 | | - }; |
|---|
| 107 | + |
|---|
| 108 | +#ifdef CONFIG_MMU_NOTIFIER |
|---|
| 109 | + struct mmu_interval_notifier notifier; |
|---|
| 110 | +#endif |
|---|
| 111 | + |
|---|
| 112 | + struct list_head shadow_list; |
|---|
| 106 | 113 | |
|---|
| 107 | 114 | struct kgd_mem *kfd_bo; |
|---|
| 108 | 115 | }; |
|---|
| .. | .. |
|---|
| 190 | 197 | */ |
|---|
| 191 | 198 | static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo) |
|---|
| 192 | 199 | { |
|---|
| 193 | | - return drm_vma_node_offset_addr(&bo->tbo.vma_node); |
|---|
| 194 | | -} |
|---|
| 195 | | - |
|---|
| 196 | | -/** |
|---|
| 197 | | - * amdgpu_bo_gpu_accessible - return whether the bo is currently in memory that |
|---|
| 198 | | - * is accessible to the GPU. |
|---|
| 199 | | - */ |
|---|
| 200 | | -static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo) |
|---|
| 201 | | -{ |
|---|
| 202 | | - switch (bo->tbo.mem.mem_type) { |
|---|
| 203 | | - case TTM_PL_TT: return amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem); |
|---|
| 204 | | - case TTM_PL_VRAM: return true; |
|---|
| 205 | | - default: return false; |
|---|
| 206 | | - } |
|---|
| 200 | + return drm_vma_node_offset_addr(&bo->tbo.base.vma_node); |
|---|
| 207 | 201 | } |
|---|
| 208 | 202 | |
|---|
| 209 | 203 | /** |
|---|
| .. | .. |
|---|
| 235 | 229 | return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC; |
|---|
| 236 | 230 | } |
|---|
| 237 | 231 | |
|---|
| 232 | +/** |
|---|
| 233 | + * amdgpu_bo_encrypted - test if the BO is encrypted |
|---|
| 234 | + * @bo: pointer to a buffer object |
|---|
| 235 | + * |
|---|
| 236 | + * Return true if the buffer object is encrypted, false otherwise. |
|---|
| 237 | + */ |
|---|
| 238 | +static inline bool amdgpu_bo_encrypted(struct amdgpu_bo *bo) |
|---|
| 239 | +{ |
|---|
| 240 | + return bo->flags & AMDGPU_GEM_CREATE_ENCRYPTED; |
|---|
| 241 | +} |
|---|
| 242 | + |
|---|
| 238 | 243 | bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); |
|---|
| 239 | 244 | void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain); |
|---|
| 240 | 245 | |
|---|
| .. | .. |
|---|
| 249 | 254 | unsigned long size, int align, |
|---|
| 250 | 255 | u32 domain, struct amdgpu_bo **bo_ptr, |
|---|
| 251 | 256 | u64 *gpu_addr, void **cpu_addr); |
|---|
| 257 | +int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, |
|---|
| 258 | + uint64_t offset, uint64_t size, uint32_t domain, |
|---|
| 259 | + struct amdgpu_bo **bo_ptr, void **cpu_addr); |
|---|
| 252 | 260 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, |
|---|
| 253 | 261 | void **cpu_addr); |
|---|
| 254 | 262 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr); |
|---|
| .. | .. |
|---|
| 275 | 283 | uint64_t *flags); |
|---|
| 276 | 284 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, |
|---|
| 277 | 285 | bool evict, |
|---|
| 278 | | - struct ttm_mem_reg *new_mem); |
|---|
| 286 | + struct ttm_resource *new_mem); |
|---|
| 287 | +void amdgpu_bo_release_notify(struct ttm_buffer_object *bo); |
|---|
| 279 | 288 | int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); |
|---|
| 280 | 289 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
|---|
| 281 | 290 | bool shared); |
|---|
| 291 | +int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, |
|---|
| 292 | + enum amdgpu_sync_mode sync_mode, void *owner, |
|---|
| 293 | + bool intr); |
|---|
| 294 | +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr); |
|---|
| 282 | 295 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); |
|---|
| 283 | | -int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, |
|---|
| 284 | | - struct amdgpu_ring *ring, |
|---|
| 285 | | - struct amdgpu_bo *bo, |
|---|
| 286 | | - struct reservation_object *resv, |
|---|
| 287 | | - struct dma_fence **fence, bool direct); |
|---|
| 296 | +u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo); |
|---|
| 288 | 297 | int amdgpu_bo_validate(struct amdgpu_bo *bo); |
|---|
| 289 | | -int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, |
|---|
| 290 | | - struct amdgpu_ring *ring, |
|---|
| 291 | | - struct amdgpu_bo *bo, |
|---|
| 292 | | - struct reservation_object *resv, |
|---|
| 293 | | - struct dma_fence **fence, |
|---|
| 294 | | - bool direct); |
|---|
| 298 | +int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, |
|---|
| 299 | + struct dma_fence **fence); |
|---|
| 295 | 300 | uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, |
|---|
| 296 | 301 | uint32_t domain); |
|---|
| 297 | 302 | |
|---|
| .. | .. |
|---|
| 326 | 331 | void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, |
|---|
| 327 | 332 | struct seq_file *m); |
|---|
| 328 | 333 | #endif |
|---|
| 334 | +int amdgpu_debugfs_sa_init(struct amdgpu_device *adev); |
|---|
| 335 | + |
|---|
| 336 | +bool amdgpu_bo_support_uswc(u64 bo_flags); |
|---|
| 329 | 337 | |
|---|
| 330 | 338 | |
|---|
| 331 | 339 | #endif |
|---|