disable FB

hc

2024-01-04 1543e317f1da31b75942316931e8f491a8920811

 kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
..
..
@@ -27,14 +27,14 @@
27
27
 
28
28
 #include <linux/types.h>
29
29
 #include <linux/mm.h>
30
-#include <linux/mmu_context.h>
30
+#include <linux/kthread.h>
31
31
 #include <linux/workqueue.h>
32
32
 #include <kgd_kfd_interface.h>
33
33
 #include <drm/ttm/ttm_execbuf_util.h>
34
34
 #include "amdgpu_sync.h"
35
35
 #include "amdgpu_vm.h"
36
36
 
37
-extern const struct kgd2kfd_calls *kgd2kfd;
37
+extern uint64_t amdgpu_amdkfd_total_mem_size;
38
38
 
39
39
 struct amdgpu_device;
40
40
 
..
..
@@ -58,15 +58,15 @@
58
58
 	unsigned int mapped_to_gpu_memory;
59
59
 	uint64_t va;
60
60
 
61
-	uint32_t mapping_flags;
61
+	uint32_t alloc_flags;
62
62
 
63
63
 	atomic_t invalid;
64
64
 	struct amdkfd_process_info *process_info;
65
-	struct page **user_pages;
66
65
 
67
66
 	struct amdgpu_sync sync;
68
67
 
69
68
 	bool aql_queue;
69
+	bool is_imported;
70
70
 };
71
71
 
72
72
 /* KFD Memory Eviction */
..
..
@@ -77,10 +77,23 @@
77
77
 	char timeline_name[TASK_COMM_LEN];
78
78
 };
79
79
 
80
-struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
81
-						       struct mm_struct *mm);
82
-bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
83
-struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
80
+struct amdgpu_kfd_dev {
81
+	struct kfd_dev *dev;
82
+	uint64_t vram_used;
83
+};
84
+
85
+enum kgd_engine_type {
86
+	KGD_ENGINE_PFP = 1,
87
+	KGD_ENGINE_ME,
88
+	KGD_ENGINE_CE,
89
+	KGD_ENGINE_MEC1,
90
+	KGD_ENGINE_MEC2,
91
+	KGD_ENGINE_RLC,
92
+	KGD_ENGINE_SDMA1,
93
+	KGD_ENGINE_SDMA2,
94
+	KGD_ENGINE_MAX
95
+};
96
+
84
97
 
85
98
 struct amdkfd_process_info {
86
99
 	/* List head of all VMs that belong to a KFD process */
..
..
@@ -107,23 +120,21 @@
107
120
 int amdgpu_amdkfd_init(void);
108
121
 void amdgpu_amdkfd_fini(void);
109
122
 
110
-void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
111
-int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
123
+void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);
124
+int amdgpu_amdkfd_resume_iommu(struct amdgpu_device *adev);
125
+int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);
112
126
 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
113
127
 			const void *ih_ring_entry);
114
128
 void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
115
129
 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
116
130
 void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
117
-
118
-int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
119
131
 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
120
132
 				uint32_t vmid, uint64_t gpu_addr,
121
133
 				uint32_t *ib_cmd, uint32_t ib_len);
122
134
 void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
123
-
124
-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
125
-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
126
-struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
135
+bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
136
+int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid);
137
+int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid);
127
138
 
128
139
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
129
140
 
..
..
@@ -133,52 +144,112 @@
133
144
 
134
145
 void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
135
146
 
147
+int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
148
+					int queue_bit);
149
+
150
+struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
151
+								struct mm_struct *mm);
152
+#if IS_ENABLED(CONFIG_HSA_AMD)
153
+bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
154
+struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
155
+int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo);
156
+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
157
+#else
158
+static inline
159
+bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
160
+{
161
+	return false;
162
+}
163
+
164
+static inline
165
+struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
166
+{
167
+	return NULL;
168
+}
169
+
170
+static inline
171
+int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
172
+{
173
+	return 0;
174
+}
175
+
176
+static inline
177
+int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
178
+{
179
+	return 0;
180
+}
181
+#endif
136
182
 /* Shared API */
137
-int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
138
-			void **mem_obj, uint64_t *gpu_addr,
139
-			void **cpu_ptr, bool mqd_gfx9);
140
-void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
141
-void get_local_mem_info(struct kgd_dev *kgd,
142
-			struct kfd_local_mem_info *mem_info);
143
-uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
183
+int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
184
+				void **mem_obj, uint64_t *gpu_addr,
185
+				void **cpu_ptr, bool mqd_gfx9);
186
+void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
187
+int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, void **mem_obj);
188
+void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj);
189
+int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);
190
+int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
191
+uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
192
+				      enum kgd_engine_type type);
193
+void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
194
+				      struct kfd_local_mem_info *mem_info);
195
+uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
144
196
 
145
-uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
146
-void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
197
+uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
198
+void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
199
+int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
200
+				  struct kgd_dev **dmabuf_kgd,
201
+				  uint64_t *bo_size, void *metadata_buffer,
202
+				  size_t buffer_size, uint32_t *metadata_size,
203
+				  uint32_t *flags);
147
204
 uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
205
+uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
206
+uint64_t amdgpu_amdkfd_get_unique_id(struct kgd_dev *kgd);
207
+uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
208
+uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
209
+uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
210
+int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
211
+uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
148
212
 
213
+/* Read user wptr from a specified user address space with page fault
214
+ * disabled. The memory must be pinned and mapped to the hardware when
215
+ * this is called in hqd_load functions, so it should never fault in
216
+ * the first place. This resolves a circular lock dependency involving
217
+ * four locks, including the DQM lock and mmap_lock.
218
+ */
149
219
 #define read_user_wptr(mmptr, wptr, dst)				\
150
220
 	({								\
151
221
 		bool valid = false;					\
152
222
 		if ((mmptr) && (wptr)) {				\
223
+			pagefault_disable();				\
153
224
 			if ((mmptr) == current->mm) {			\
154
225
 				valid = !get_user((dst), (wptr));	\
155
-			} else if (current->mm == NULL) {		\
156
-				use_mm(mmptr);				\
226
+			} else if (current->flags & PF_KTHREAD) {	\
227
+				kthread_use_mm(mmptr);			\
157
228
 				valid = !get_user((dst), (wptr));	\
158
-				unuse_mm(mmptr);			\
229
+				kthread_unuse_mm(mmptr);		\
159
230
 			}						\
231
+			pagefault_enable();				\
160
232
 		}							\
161
233
 		valid;							\
162
234
 	})
163
235
 
164
236
 /* GPUVM API */
165
-int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
166
-					void **process_info,
167
-					struct dma_fence **ef);
168
-int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
169
-					struct file *filp,
237
+int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, u32 pasid,
170
238
 					void **vm, void **process_info,
171
239
 					struct dma_fence **ef);
172
-void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
173
-				struct amdgpu_vm *vm);
240
+int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
241
+					struct file *filp, u32 pasid,
242
+					void **vm, void **process_info,
243
+					struct dma_fence **ef);
174
244
 void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
175
-uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
245
+void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm);
246
+uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
176
247
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
177
248
 		struct kgd_dev *kgd, uint64_t va, uint64_t size,
178
249
 		void *vm, struct kgd_mem **mem,
179
250
 		uint64_t *offset, uint32_t flags);
180
251
 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
181
-		struct kgd_dev *kgd, struct kgd_mem *mem);
252
+		struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size);
182
253
 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
183
254
 		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
184
255
 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
..
..
@@ -189,11 +260,124 @@
189
260
 		struct kgd_mem *mem, void **kptr, uint64_t *size);
190
261
 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
191
262
 					    struct dma_fence **ef);
192
-
193
263
 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
194
264
 					      struct kfd_vm_fault_info *info);
195
-
265
+int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
266
+				      struct dma_buf *dmabuf,
267
+				      uint64_t va, void *vm,
268
+				      struct kgd_mem **mem, uint64_t *size,
269
+				      uint64_t *mmap_offset);
270
+int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
271
+				struct tile_config *config);
272
+#if IS_ENABLED(CONFIG_HSA_AMD)
196
273
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
197
-void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
274
+void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
275
+				struct amdgpu_vm *vm);
276
+void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
277
+#else
278
+static inline
279
+void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
280
+{
281
+}
198
282
 
283
+static inline
284
+void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
285
+					struct amdgpu_vm *vm)
286
+{
287
+}
288
+
289
+static inline
290
+void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)
291
+{
292
+}
293
+#endif
294
+/* KGD2KFD callbacks */
295
+int kgd2kfd_quiesce_mm(struct mm_struct *mm);
296
+int kgd2kfd_resume_mm(struct mm_struct *mm);
297
+int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
298
+						struct dma_fence *fence);
299
+#if IS_ENABLED(CONFIG_HSA_AMD)
300
+int kgd2kfd_init(void);
301
+void kgd2kfd_exit(void);
302
+struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
303
+			      unsigned int asic_type, bool vf);
304
+bool kgd2kfd_device_init(struct kfd_dev *kfd,
305
+			 struct drm_device *ddev,
306
+			 const struct kgd2kfd_shared_resources *gpu_resources);
307
+void kgd2kfd_device_exit(struct kfd_dev *kfd);
308
+void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm);
309
+int kgd2kfd_resume_iommu(struct kfd_dev *kfd);
310
+int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm);
311
+int kgd2kfd_pre_reset(struct kfd_dev *kfd);
312
+int kgd2kfd_post_reset(struct kfd_dev *kfd);
313
+void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
314
+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);
315
+void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask);
316
+#else
317
+static inline int kgd2kfd_init(void)
318
+{
319
+	return -ENOENT;
320
+}
321
+
322
+static inline void kgd2kfd_exit(void)
323
+{
324
+}
325
+
326
+static inline
327
+struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
328
+					unsigned int asic_type, bool vf)
329
+{
330
+	return NULL;
331
+}
332
+
333
+static inline
334
+bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev,
335
+				const struct kgd2kfd_shared_resources *gpu_resources)
336
+{
337
+	return false;
338
+}
339
+
340
+static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)
341
+{
342
+}
343
+
344
+static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
345
+{
346
+}
347
+
348
+static int __maybe_unused kgd2kfd_resume_iommu(struct kfd_dev *kfd)
349
+{
350
+	return 0;
351
+}
352
+
353
+static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
354
+{
355
+	return 0;
356
+}
357
+
358
+static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd)
359
+{
360
+	return 0;
361
+}
362
+
363
+static inline int kgd2kfd_post_reset(struct kfd_dev *kfd)
364
+{
365
+	return 0;
366
+}
367
+
368
+static inline
369
+void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
370
+{
371
+}
372
+
373
+static inline
374
+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
375
+{
376
+}
377
+
378
+static inline
379
+void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask)
380
+{
381
+}
382
+#endif
199
383
 #endif /* AMDGPU_AMDKFD_H_INCLUDED */