change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
..
..
@@ -31,6 +31,7 @@
31
31
 #include "gca/gfx_8_0_sh_mask.h"
32
32
 #include "gca/gfx_8_0_enum.h"
33
33
 #include "oss/oss_3_0_sh_mask.h"
34
+
34
35
 #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
35
36
 
36
37
 static inline struct vi_mqd *get_mqd(void *mqd)
..
..
@@ -68,21 +69,33 @@
68
69
 		m->compute_static_thread_mgmt_se3);
69
70
 }
70
71
 
71
-static int init_mqd(struct mqd_manager *mm, void **mqd,
72
-			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
72
+static void set_priority(struct vi_mqd *m, struct queue_properties *q)
73
+{
74
+	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
75
+	m->cp_hqd_queue_priority = q->priority;
76
+}
77
+
78
+static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
79
+					struct queue_properties *q)
80
+{
81
+	struct kfd_mem_obj *mqd_mem_obj;
82
+
83
+	if (kfd_gtt_sa_allocate(kfd, sizeof(struct vi_mqd),
84
+			&mqd_mem_obj))
85
+		return NULL;
86
+
87
+	return mqd_mem_obj;
88
+}
89
+
90
+static void init_mqd(struct mqd_manager *mm, void **mqd,
91
+			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
73
92
 			struct queue_properties *q)
74
93
 {
75
-	int retval;
76
94
 	uint64_t addr;
77
95
 	struct vi_mqd *m;
78
96
 
79
-	retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd),
80
-			mqd_mem_obj);
81
-	if (retval != 0)
82
-		return -ENOMEM;
83
-
84
-	m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr;
85
-	addr = (*mqd_mem_obj)->gpu_addr;
97
+	m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr;
98
+	addr = mqd_mem_obj->gpu_addr;
86
99
 
87
100
 	memset(m, 0, sizeof(struct vi_mqd));
88
101
 
..
..
@@ -104,11 +117,9 @@
104
117
 
105
118
 	m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
106
119
 			1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
107
-			10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
120
+			1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
108
121
 
109
-	m->cp_hqd_pipe_priority = 1;
110
-	m->cp_hqd_queue_priority = 15;
111
-
122
+	set_priority(m, q);
112
123
 	m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
113
124
 
114
125
 	if (q->format == KFD_QUEUE_FORMAT_AQL)
..
..
@@ -139,9 +150,7 @@
139
150
 	*mqd = m;
140
151
 	if (gart_addr)
141
152
 		*gart_addr = addr;
142
-	retval = mm->update_mqd(mm, m, q);
143
-
144
-	return retval;
153
+	mm->update_mqd(mm, m, q);
145
154
 }
146
155
 
147
156
 static int load_mqd(struct mqd_manager *mm, void *mqd,
..
..
@@ -157,7 +166,7 @@
157
166
 					  wptr_shift, wptr_mask, mms);
158
167
 }
159
168
 
160
-static int __update_mqd(struct mqd_manager *mm, void *mqd,
169
+static void __update_mqd(struct mqd_manager *mm, void *mqd,
161
170
 			struct queue_properties *q, unsigned int mtype,
162
171
 			unsigned int atc_bit)
163
172
 {
..
..
@@ -222,26 +231,22 @@
222
231
 			mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
223
232
 
224
233
 	update_cu_mask(mm, mqd, q);
234
+	set_priority(m, q);
225
235
 
226
-	q->is_active = (q->queue_size > 0 &&
227
-			q->queue_address != 0 &&
228
-			q->queue_percent > 0 &&
229
-			!q->is_evicted);
230
-
231
-	return 0;
236
+	q->is_active = QUEUE_IS_ACTIVE(*q);
232
237
 }
233
238
 
234
239
 
235
-static int update_mqd(struct mqd_manager *mm, void *mqd,
240
+static void update_mqd(struct mqd_manager *mm, void *mqd,
236
241
 			struct queue_properties *q)
237
242
 {
238
-	return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
243
+	__update_mqd(mm, mqd, q, MTYPE_CC, 1);
239
244
 }
240
245
 
241
-static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
246
+static void update_mqd_tonga(struct mqd_manager *mm, void *mqd,
242
247
 			struct queue_properties *q)
243
248
 {
244
-	return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
249
+	__update_mqd(mm, mqd, q, MTYPE_UC, 0);
245
250
 }
246
251
 
247
252
 static int destroy_mqd(struct mqd_manager *mm, void *mqd,
..
..
@@ -254,7 +259,7 @@
254
259
 		pipe_id, queue_id);
255
260
 }
256
261
 
257
-static void uninit_mqd(struct mqd_manager *mm, void *mqd,
262
+static void free_mqd(struct mqd_manager *mm, void *mqd,
258
263
 			struct kfd_mem_obj *mqd_mem_obj)
259
264
 {
260
265
 	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
..
..
@@ -269,70 +274,62 @@
269
274
 		pipe_id, queue_id);
270
275
 }
271
276
 
272
-static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
273
-			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
277
+static int get_wave_state(struct mqd_manager *mm, void *mqd,
278
+			  void __user *ctl_stack,
279
+			  u32 *ctl_stack_used_size,
280
+			  u32 *save_area_used_size)
281
+{
282
+	struct vi_mqd *m;
283
+
284
+	m = get_mqd(mqd);
285
+
286
+	*ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
287
+		m->cp_hqd_cntl_stack_offset;
288
+	*save_area_used_size = m->cp_hqd_wg_state_offset -
289
+		m->cp_hqd_cntl_stack_size;
290
+
291
+	/* Control stack is not copied to user mode for GFXv8 because
292
+	 * it's part of the context save area that is already
293
+	 * accessible to user mode
294
+	 */
295
+
296
+	return 0;
297
+}
298
+
299
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
300
+			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
274
301
 			struct queue_properties *q)
275
302
 {
276
303
 	struct vi_mqd *m;
277
-	int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
278
-
279
-	if (retval != 0)
280
-		return retval;
304
+	init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
281
305
 
282
306
 	m = get_mqd(*mqd);
283
307
 
284
308
 	m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
285
309
 			1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
286
-
287
-	return retval;
288
310
 }
289
311
 
290
-static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
312
+static void update_mqd_hiq(struct mqd_manager *mm, void *mqd,
291
313
 			struct queue_properties *q)
292
314
 {
293
-	struct vi_mqd *m;
294
-	int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0);
295
-
296
-	if (retval != 0)
297
-		return retval;
298
-
299
-	m = get_mqd(mqd);
300
-	m->cp_hqd_vmid = q->vmid;
301
-	return retval;
315
+	__update_mqd(mm, mqd, q, MTYPE_UC, 0);
302
316
 }
303
317
 
304
-static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
305
-		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
318
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
319
+		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
306
320
 		struct queue_properties *q)
307
321
 {
308
-	int retval;
309
322
 	struct vi_sdma_mqd *m;
310
323
 
311
-
312
-	retval = kfd_gtt_sa_allocate(mm->dev,
313
-			sizeof(struct vi_sdma_mqd),
314
-			mqd_mem_obj);
315
-
316
-	if (retval != 0)
317
-		return -ENOMEM;
318
-
319
-	m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
324
+	m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr;
320
325
 
321
326
 	memset(m, 0, sizeof(struct vi_sdma_mqd));
322
327
 
323
328
 	*mqd = m;
324
-	if (gart_addr != NULL)
325
-		*gart_addr = (*mqd_mem_obj)->gpu_addr;
329
+	if (gart_addr)
330
+		*gart_addr = mqd_mem_obj->gpu_addr;
326
331
 
327
-	retval = mm->update_mqd(mm, m, q);
328
-
329
-	return retval;
330
-}
331
-
332
-static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
333
-		struct kfd_mem_obj *mqd_mem_obj)
334
-{
335
-	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
332
+	mm->update_mqd(mm, m, q);
336
333
 }
337
334
 
338
335
 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
..
..
@@ -344,7 +341,7 @@
344
341
 					       mms);
345
342
 }
346
343
 
347
-static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
344
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
348
345
 		struct queue_properties *q)
349
346
 {
350
347
 	struct vi_sdma_mqd *m;
..
..
@@ -368,12 +365,7 @@
368
365
 	m->sdma_engine_id = q->sdma_engine_id;
369
366
 	m->sdma_queue_id = q->sdma_queue_id;
370
367
 
371
-	q->is_active = (q->queue_size > 0 &&
372
-			q->queue_address != 0 &&
373
-			q->queue_percent > 0 &&
374
-			!q->is_evicted);
375
-
376
-	return 0;
368
+	q->is_active = QUEUE_IS_ACTIVE(*q);
377
369
 }
378
370
 
379
371
 /*
..
..
@@ -429,35 +421,54 @@
429
421
 
430
422
 	switch (type) {
431
423
 	case KFD_MQD_TYPE_CP:
432
-	case KFD_MQD_TYPE_COMPUTE:
424
+		mqd->allocate_mqd = allocate_mqd;
433
425
 		mqd->init_mqd = init_mqd;
434
-		mqd->uninit_mqd = uninit_mqd;
426
+		mqd->free_mqd = free_mqd;
435
427
 		mqd->load_mqd = load_mqd;
436
428
 		mqd->update_mqd = update_mqd;
437
429
 		mqd->destroy_mqd = destroy_mqd;
438
430
 		mqd->is_occupied = is_occupied;
431
+		mqd->get_wave_state = get_wave_state;
432
+		mqd->mqd_size = sizeof(struct vi_mqd);
439
433
 #if defined(CONFIG_DEBUG_FS)
440
434
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
441
435
 #endif
442
436
 		break;
443
437
 	case KFD_MQD_TYPE_HIQ:
438
+		mqd->allocate_mqd = allocate_hiq_mqd;
444
439
 		mqd->init_mqd = init_mqd_hiq;
445
-		mqd->uninit_mqd = uninit_mqd;
440
+		mqd->free_mqd = free_mqd_hiq_sdma;
446
441
 		mqd->load_mqd = load_mqd;
447
442
 		mqd->update_mqd = update_mqd_hiq;
448
443
 		mqd->destroy_mqd = destroy_mqd;
449
444
 		mqd->is_occupied = is_occupied;
445
+		mqd->mqd_size = sizeof(struct vi_mqd);
446
+#if defined(CONFIG_DEBUG_FS)
447
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
448
+#endif
449
+		break;
450
+	case KFD_MQD_TYPE_DIQ:
451
+		mqd->allocate_mqd = allocate_mqd;
452
+		mqd->init_mqd = init_mqd_hiq;
453
+		mqd->free_mqd = free_mqd;
454
+		mqd->load_mqd = load_mqd;
455
+		mqd->update_mqd = update_mqd_hiq;
456
+		mqd->destroy_mqd = destroy_mqd;
457
+		mqd->is_occupied = is_occupied;
458
+		mqd->mqd_size = sizeof(struct vi_mqd);
450
459
 #if defined(CONFIG_DEBUG_FS)
451
460
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
452
461
 #endif
453
462
 		break;
454
463
 	case KFD_MQD_TYPE_SDMA:
464
+		mqd->allocate_mqd = allocate_sdma_mqd;
455
465
 		mqd->init_mqd = init_mqd_sdma;
456
-		mqd->uninit_mqd = uninit_mqd_sdma;
466
+		mqd->free_mqd = free_mqd_hiq_sdma;
457
467
 		mqd->load_mqd = load_mqd_sdma;
458
468
 		mqd->update_mqd = update_mqd_sdma;
459
469
 		mqd->destroy_mqd = destroy_mqd_sdma;
460
470
 		mqd->is_occupied = is_occupied_sdma;
471
+		mqd->mqd_size = sizeof(struct vi_sdma_mqd);
461
472
 #if defined(CONFIG_DEBUG_FS)
462
473
 		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
463
474
 #endif
..
..
@@ -478,7 +489,7 @@
478
489
 	mqd = mqd_manager_init_vi(type, dev);
479
490
 	if (!mqd)
480
491
 		return NULL;
481
-	if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
492
+	if (type == KFD_MQD_TYPE_CP)
482
493
 		mqd->update_mqd = update_mqd_tonga;
483
494
 	return mqd;
484
495
 }