change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
..
..
@@ -26,6 +26,7 @@
26
26
 #include "kfd_device_queue_manager.h"
27
27
 #include "kfd_priv.h"
28
28
 #include "kfd_kernel_queue.h"
29
+#include "amdgpu_amdkfd.h"
29
30
 
30
31
 static inline struct process_queue_node *get_queue_by_qid(
31
32
 			struct process_queue_manager *pqm, unsigned int qid)
..
..
@@ -52,7 +53,7 @@
52
53
 	pr_debug("The new slot id %lu\n", found);
53
54
 
54
55
 	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
55
-		pr_info("Cannot open more queues for process with pasid %d\n",
56
+		pr_info("Cannot open more queues for process with pasid 0x%x\n",
56
57
 				pqm->process->pasid);
57
58
 		return -ENOMEM;
58
59
 	}
..
..
@@ -72,6 +73,55 @@
72
73
 
73
74
 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
74
75
 	pdd->already_dequeued = true;
76
+}
77
+
78
+int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
79
+			void *gws)
80
+{
81
+	struct kfd_dev *dev = NULL;
82
+	struct process_queue_node *pqn;
83
+	struct kfd_process_device *pdd;
84
+	struct kgd_mem *mem = NULL;
85
+	int ret;
86
+
87
+	pqn = get_queue_by_qid(pqm, qid);
88
+	if (!pqn) {
89
+		pr_err("Queue id does not match any known queue\n");
90
+		return -EINVAL;
91
+	}
92
+
93
+	if (pqn->q)
94
+		dev = pqn->q->device;
95
+	if (WARN_ON(!dev))
96
+		return -ENODEV;
97
+
98
+	pdd = kfd_get_process_device_data(dev, pqm->process);
99
+	if (!pdd) {
100
+		pr_err("Process device data doesn't exist\n");
101
+		return -EINVAL;
102
+	}
103
+
104
+	/* Only allow one queue per process can have GWS assigned */
105
+	if (gws && pdd->qpd.num_gws)
106
+		return -EBUSY;
107
+
108
+	if (!gws && pdd->qpd.num_gws == 0)
109
+		return -EINVAL;
110
+
111
+	if (gws)
112
+		ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
113
+			gws, &mem);
114
+	else
115
+		ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
116
+			pqn->q->gws);
117
+	if (unlikely(ret))
118
+		return ret;
119
+
120
+	pqn->q->gws = mem;
121
+	pdd->qpd.num_gws = gws ? amdgpu_amdkfd_get_num_gws(dev->kgd) : 0;
122
+
123
+	return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
124
+							pqn->q);
75
125
 }
76
126
 
77
127
 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
..
..
@@ -100,6 +150,10 @@
100
150
 	struct process_queue_node *pqn, *next;
101
151
 
102
152
 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
153
+		if (pqn->q && pqn->q->gws)
154
+			amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
155
+				pqn->q->gws);
156
+		kfd_procfs_del_queue(pqn->q);
103
157
 		uninit_queue(pqn->q);
104
158
 		list_del(&pqn->process_queue_list);
105
159
 		kfree(pqn);
..
..
@@ -109,7 +163,7 @@
109
163
 	pqm->queue_slot_bitmap = NULL;
110
164
 }
111
165
 
112
-static int create_cp_queue(struct process_queue_manager *pqm,
166
+static int init_user_queue(struct process_queue_manager *pqm,
113
167
 				struct kfd_dev *dev, struct queue **q,
114
168
 				struct queue_properties *q_properties,
115
169
 				struct file *f, unsigned int qid)
..
..
@@ -139,7 +193,8 @@
139
193
 			    struct kfd_dev *dev,
140
194
 			    struct file *f,
141
195
 			    struct queue_properties *properties,
142
-			    unsigned int *qid)
196
+			    unsigned int *qid,
197
+			    uint32_t *p_doorbell_offset_in_process)
143
198
 {
144
199
 	int retval;
145
200
 	struct kfd_process_device *pdd;
..
..
@@ -186,19 +241,19 @@
186
241
 
187
242
 	switch (type) {
188
243
 	case KFD_QUEUE_TYPE_SDMA:
189
-		if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
190
-			pr_err("Over-subscription is not allowed for SDMA.\n");
191
-			retval = -EPERM;
192
-			goto err_create_queue;
193
-		}
194
-
195
-		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
244
+	case KFD_QUEUE_TYPE_SDMA_XGMI:
245
+		/* SDMA queues are always allocated statically no matter
246
+		 * which scheduler mode is used. We also do not need to
247
+		 * check whether a SDMA queue can be allocated here, because
248
+		 * allocate_sdma_queue() in create_queue() has the
249
+		 * corresponding check logic.
250
+		 */
251
+		retval = init_user_queue(pqm, dev, &q, properties, f, *qid);
196
252
 		if (retval != 0)
197
253
 			goto err_create_queue;
198
254
 		pqn->q = q;
199
255
 		pqn->kq = NULL;
200
256
 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
201
-		pr_debug("DQM returned %d for create_queue\n", retval);
202
257
 		print_queue(q);
203
258
 		break;
204
259
 
..
..
@@ -207,19 +262,18 @@
207
262
 		if ((dev->dqm->sched_policy ==
208
263
 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
209
264
 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
210
-		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
265
+		(dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {
211
266
 			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
212
267
 			retval = -EPERM;
213
268
 			goto err_create_queue;
214
269
 		}
215
270
 
216
-		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
271
+		retval = init_user_queue(pqm, dev, &q, properties, f, *qid);
217
272
 		if (retval != 0)
218
273
 			goto err_create_queue;
219
274
 		pqn->q = q;
220
275
 		pqn->kq = NULL;
221
276
 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
222
-		pr_debug("DQM returned %d for create_queue\n", retval);
223
277
 		print_queue(q);
224
278
 		break;
225
279
 	case KFD_QUEUE_TYPE_DIQ:
..
..
@@ -240,17 +294,20 @@
240
294
 	}
241
295
 
242
296
 	if (retval != 0) {
243
-		pr_err("Pasid %d DQM create queue %d failed. ret %d\n",
297
+		pr_err("Pasid 0x%x DQM create queue type %d failed. ret %d\n",
244
298
 			pqm->process->pasid, type, retval);
245
299
 		goto err_create_queue;
246
300
 	}
247
301
 
248
-	if (q)
302
+	if (q && p_doorbell_offset_in_process)
249
303
 		/* Return the doorbell offset within the doorbell page
250
304
 		 * to the caller so it can be passed up to user mode
251
305
 		 * (in bytes).
306
+		 * There are always 1024 doorbells per process, so in case
307
+		 * of 8-byte doorbells, there are two doorbell pages per
308
+		 * process.
252
309
 		 */
253
-		properties->doorbell_off =
310
+		*p_doorbell_offset_in_process =
254
311
 			(q->properties.doorbell_off * sizeof(uint32_t)) &
255
312
 			(kfd_doorbell_process_slice(dev) - 1);
256
313
 
..
..
@@ -260,12 +317,16 @@
260
317
 
261
318
 	if (q) {
262
319
 		pr_debug("PQM done creating queue\n");
320
+		kfd_procfs_add_queue(q);
263
321
 		print_queue_properties(&q->properties);
264
322
 	}
265
323
 
266
324
 	return retval;
267
325
 
268
326
 err_create_queue:
327
+	uninit_queue(q);
328
+	if (kq)
329
+		kernel_queue_uninit(kq, false);
269
330
 	kfree(pqn);
270
331
 err_allocate_pqn:
271
332
 	/* check if queues list is empty unregister process from device */
..
..
@@ -312,19 +373,27 @@
312
373
 		/* destroy kernel queue (DIQ) */
313
374
 		dqm = pqn->kq->dev->dqm;
314
375
 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
315
-		kernel_queue_uninit(pqn->kq);
376
+		kernel_queue_uninit(pqn->kq, false);
316
377
 	}
317
378
 
318
379
 	if (pqn->q) {
380
+		kfd_procfs_del_queue(pqn->q);
319
381
 		dqm = pqn->q->device->dqm;
320
382
 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
321
383
 		if (retval) {
322
-			pr_err("Pasid %d destroy queue %d failed, ret %d\n",
384
+			pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",
323
385
 				pqm->process->pasid,
324
386
 				pqn->q->properties.queue_id, retval);
325
387
 			if (retval != -ETIME)
326
388
 				goto err_destroy_queue;
327
389
 		}
390
+
391
+		if (pqn->q->gws) {
392
+			amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info,
393
+				pqn->q->gws);
394
+			pdd->qpd.num_gws = 0;
395
+		}
396
+
328
397
 		kfree(pqn->q->properties.cu_mask);
329
398
 		pqn->q->properties.cu_mask = NULL;
330
399
 		uninit_queue(pqn->q);
..
..
@@ -408,6 +477,37 @@
408
477
 	return NULL;
409
478
 }
410
479
 
480
+struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
481
+					unsigned int qid)
482
+{
483
+	struct process_queue_node *pqn;
484
+
485
+	pqn = get_queue_by_qid(pqm, qid);
486
+	return pqn ? pqn->q : NULL;
487
+}
488
+
489
+int pqm_get_wave_state(struct process_queue_manager *pqm,
490
+		       unsigned int qid,
491
+		       void __user *ctl_stack,
492
+		       u32 *ctl_stack_used_size,
493
+		       u32 *save_area_used_size)
494
+{
495
+	struct process_queue_node *pqn;
496
+
497
+	pqn = get_queue_by_qid(pqm, qid);
498
+	if (!pqn) {
499
+		pr_debug("amdkfd: No queue %d exists for operation\n",
500
+			 qid);
501
+		return -EFAULT;
502
+	}
503
+
504
+	return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
505
+						       pqn->q,
506
+						       ctl_stack,
507
+						       ctl_stack_used_size,
508
+						       save_area_used_size);
509
+}
510
+
411
511
 #if defined(CONFIG_DEBUG_FS)
412
512
 
413
513
 int pqm_debugfs_mqds(struct seq_file *m, void *data)
..
..
@@ -424,6 +524,7 @@
424
524
 			q = pqn->q;
425
525
 			switch (q->properties.type) {
426
526
 			case KFD_QUEUE_TYPE_SDMA:
527
+			case KFD_QUEUE_TYPE_SDMA_XGMI:
427
528
 				seq_printf(m, "  SDMA queue on device %x\n",
428
529
 					   q->device->id);
429
530
 				mqd_type = KFD_MQD_TYPE_SDMA;
..
..
@@ -439,8 +540,7 @@
439
540
 					   q->properties.type, q->device->id);
440
541
 				continue;
441
542
 			}
442
-			mqd_mgr = q->device->dqm->ops.get_mqd_manager(
443
-				q->device->dqm, mqd_type);
543
+			mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
444
544
 		} else if (pqn->kq) {
445
545
 			q = pqn->kq->queue;
446
546
 			mqd_mgr = pqn->kq->mqd_mgr;
..
..
@@ -448,7 +548,6 @@
448
548
 			case KFD_QUEUE_TYPE_DIQ:
449
549
 				seq_printf(m, "  DIQ on device %x\n",
450
550
 					   pqn->kq->dev->id);
451
-				mqd_type = KFD_MQD_TYPE_HIQ;
452
551
 				break;
453
552
 			default:
454
553
 				seq_printf(m,