change system file

hc

2024-10-09 244b2c5ca8b14627e4a17755e5922221e121c771

 kernel/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
..
..
@@ -34,7 +34,10 @@
34
34
 
35
35
 #define PM4_COUNT_ZERO (((1 << 15) - 1) << 16)
36
36
 
37
-static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
37
+/* Initialize a kernel queue, including allocations of GART memory
38
+ * needed for the queue.
39
+ */
40
+static bool kq_initialize(struct kernel_queue *kq, struct kfd_dev *dev,
38
41
 		enum kfd_queue_type type, unsigned int queue_size)
39
42
 {
40
43
 	struct queue_properties prop;
..
..
@@ -58,9 +61,10 @@
58
61
 	kq->nop_packet = nop.u32all;
59
62
 	switch (type) {
60
63
 	case KFD_QUEUE_TYPE_DIQ:
64
+		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ];
65
+		break;
61
66
 	case KFD_QUEUE_TYPE_HIQ:
62
-		kq->mqd_mgr = dev->dqm->ops.get_mqd_manager(dev->dqm,
63
-						KFD_MQD_TYPE_HIQ);
67
+		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
64
68
 		break;
65
69
 	default:
66
70
 		pr_err("Invalid queue type %d\n", type);
..
..
@@ -86,9 +90,17 @@
86
90
 	kq->pq_kernel_addr = kq->pq->cpu_ptr;
87
91
 	kq->pq_gpu_addr = kq->pq->gpu_addr;
88
92
 
89
-	retval = kq->ops_asic_specific.initialize(kq, dev, type, queue_size);
90
-	if (!retval)
91
-		goto err_eop_allocate_vidmem;
93
+	/* For CIK family asics, kq->eop_mem is not needed */
94
+	if (dev->device_info->asic_family > CHIP_MULLINS) {
95
+		retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem);
96
+		if (retval != 0)
97
+			goto err_eop_allocate_vidmem;
98
+
99
+		kq->eop_gpu_addr = kq->eop_mem->gpu_addr;
100
+		kq->eop_kernel_addr = kq->eop_mem->cpu_ptr;
101
+
102
+		memset(kq->eop_kernel_addr, 0, PAGE_SIZE);
103
+	}
92
104
 
93
105
 	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel),
94
106
 					&kq->rptr_mem);
..
..
@@ -114,6 +126,7 @@
114
126
 
115
127
 	prop.queue_size = queue_size;
116
128
 	prop.is_interop = false;
129
+	prop.is_gws = false;
117
130
 	prop.priority = 1;
118
131
 	prop.queue_percent = 100;
119
132
 	prop.type = type;
..
..
@@ -131,13 +144,14 @@
131
144
 	kq->queue->device = dev;
132
145
 	kq->queue->process = kfd_get_process(current);
133
146
 
134
-	retval = kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
135
-					&kq->queue->mqd_mem_obj,
147
+	kq->queue->mqd_mem_obj = kq->mqd_mgr->allocate_mqd(kq->mqd_mgr->dev,
148
+					&kq->queue->properties);
149
+	if (!kq->queue->mqd_mem_obj)
150
+		goto err_allocate_mqd;
151
+	kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
152
+					kq->queue->mqd_mem_obj,
136
153
 					&kq->queue->gart_mqd_addr,
137
154
 					&kq->queue->properties);
138
-	if (retval != 0)
139
-		goto err_init_mqd;
140
-
141
155
 	/* assign HIQ to HQD */
142
156
 	if (type == KFD_QUEUE_TYPE_HIQ) {
143
157
 		pr_debug("Assigning hiq to hqd\n");
..
..
@@ -163,7 +177,8 @@
163
177
 
164
178
 	return true;
165
179
 err_alloc_fence:
166
-err_init_mqd:
180
+	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, kq->queue->mqd_mem_obj);
181
+err_allocate_mqd:
167
182
 	uninit_queue(kq->queue);
168
183
 err_init_queue:
169
184
 	kfd_gtt_sa_free(dev, kq->wptr_mem);
..
..
@@ -180,9 +195,10 @@
180
195
 
181
196
 }
182
197
 
183
-static void uninitialize(struct kernel_queue *kq)
198
+/* Uninitialize a kernel queue and free all its memory usages. */
199
+static void kq_uninitialize(struct kernel_queue *kq, bool hanging)
184
200
 {
185
-	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
201
+	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ && !hanging)
186
202
 		kq->mqd_mgr->destroy_mqd(kq->mqd_mgr,
187
203
 					kq->queue->mqd,
188
204
 					KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
..
..
@@ -192,19 +208,24 @@
192
208
 	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
193
209
 		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
194
210
 
195
-	kq->mqd_mgr->uninit_mqd(kq->mqd_mgr, kq->queue->mqd,
211
+	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd,
196
212
 				kq->queue->mqd_mem_obj);
197
213
 
198
214
 	kfd_gtt_sa_free(kq->dev, kq->rptr_mem);
199
215
 	kfd_gtt_sa_free(kq->dev, kq->wptr_mem);
200
-	kq->ops_asic_specific.uninitialize(kq);
216
+
217
+	/* For CIK family asics, kq->eop_mem is Null, kfd_gtt_sa_free()
218
+	 * is able to handle NULL properly.
219
+	 */
220
+	kfd_gtt_sa_free(kq->dev, kq->eop_mem);
221
+
201
222
 	kfd_gtt_sa_free(kq->dev, kq->pq);
202
223
 	kfd_release_kernel_doorbell(kq->dev,
203
224
 					kq->queue->properties.doorbell_ptr);
204
225
 	uninit_queue(kq->queue);
205
226
 }
206
227
 
207
-static int acquire_packet_buffer(struct kernel_queue *kq,
228
+int kq_acquire_packet_buffer(struct kernel_queue *kq,
208
229
 		size_t packet_size_in_dwords, unsigned int **buffer_ptr)
209
230
 {
210
231
 	size_t available_size;
..
..
@@ -265,7 +286,7 @@
265
286
 	return -ENOMEM;
266
287
 }
267
288
 
268
-static void submit_packet(struct kernel_queue *kq)
289
+void kq_submit_packet(struct kernel_queue *kq)
269
290
 {
270
291
 #ifdef DEBUG
271
292
 	int i;
..
..
@@ -277,11 +298,18 @@
277
298
 	}
278
299
 	pr_debug("\n");
279
300
 #endif
280
-
281
-	kq->ops_asic_specific.submit_packet(kq);
301
+	if (kq->dev->device_info->doorbell_size == 8) {
302
+		*kq->wptr64_kernel = kq->pending_wptr64;
303
+		write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,
304
+					kq->pending_wptr64);
305
+	} else {
306
+		*kq->wptr_kernel = kq->pending_wptr;
307
+		write_kernel_doorbell(kq->queue->properties.doorbell_ptr,
308
+					kq->pending_wptr);
309
+	}
282
310
 }
283
311
 
284
-static void rollback_packet(struct kernel_queue *kq)
312
+void kq_rollback_packet(struct kernel_queue *kq)
285
313
 {
286
314
 	if (kq->dev->device_info->doorbell_size == 8) {
287
315
 		kq->pending_wptr64 = *kq->wptr64_kernel;
..
..
@@ -301,49 +329,18 @@
301
329
 	if (!kq)
302
330
 		return NULL;
303
331
 
304
-	kq->ops.initialize = initialize;
305
-	kq->ops.uninitialize = uninitialize;
306
-	kq->ops.acquire_packet_buffer = acquire_packet_buffer;
307
-	kq->ops.submit_packet = submit_packet;
308
-	kq->ops.rollback_packet = rollback_packet;
309
-
310
-	switch (dev->device_info->asic_family) {
311
-	case CHIP_CARRIZO:
312
-	case CHIP_TONGA:
313
-	case CHIP_FIJI:
314
-	case CHIP_POLARIS10:
315
-	case CHIP_POLARIS11:
316
-		kernel_queue_init_vi(&kq->ops_asic_specific);
317
-		break;
318
-
319
-	case CHIP_KAVERI:
320
-	case CHIP_HAWAII:
321
-		kernel_queue_init_cik(&kq->ops_asic_specific);
322
-		break;
323
-
324
-	case CHIP_VEGA10:
325
-	case CHIP_RAVEN:
326
-		kernel_queue_init_v9(&kq->ops_asic_specific);
327
-		break;
328
-	default:
329
-		WARN(1, "Unexpected ASIC family %u",
330
-		     dev->device_info->asic_family);
331
-		goto out_free;
332
-	}
333
-
334
-	if (kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE))
332
+	if (kq_initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE))
335
333
 		return kq;
336
334
 
337
335
 	pr_err("Failed to init kernel queue\n");
338
336
 
339
-out_free:
340
337
 	kfree(kq);
341
338
 	return NULL;
342
339
 }
343
340
 
344
-void kernel_queue_uninit(struct kernel_queue *kq)
341
+void kernel_queue_uninit(struct kernel_queue *kq, bool hanging)
345
342
 {
346
-	kq->ops.uninitialize(kq);
343
+	kq_uninitialize(kq, hanging);
347
344
 	kfree(kq);
348
345
 }
349
346
 
..
..
@@ -363,7 +360,7 @@
363
360
 		return;
364
361
 	}
365
362
 
366
-	retval = kq->ops.acquire_packet_buffer(kq, 5, &buffer);
363
+	retval = kq_acquire_packet_buffer(kq, 5, &buffer);
367
364
 	if (unlikely(retval != 0)) {
368
365
 		pr_err("  Failed to acquire packet buffer\n");
369
366
 		pr_err("Kernel queue test failed\n");
..
..
@@ -371,7 +368,7 @@
371
368
 	}
372
369
 	for (i = 0; i < 5; i++)
373
370
 		buffer[i] = kq->nop_packet;
374
-	kq->ops.submit_packet(kq);
371
+	kq_submit_packet(kq);
375
372
 
376
373
 	pr_err("Ending kernel queue test\n");
377
374
 }