forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
....@@ -45,393 +45,81 @@
4545 
4646 #include <linux/firmware.h>
4747 #include <linux/module.h>
48
-#include <linux/mmu_notifier.h>
49
-#include <linux/interval_tree.h>
50
-#include <drm/drmP.h>
5148 #include <drm/drm.h>
5249 
5350 #include "amdgpu.h"
5451 #include "amdgpu_amdkfd.h"
5552 
5653 /**
57
- * struct amdgpu_mn
54
+ * amdgpu_mn_invalidate_gfx - callback to notify about mm change
5855  *
59
- * @adev: amdgpu device pointer
60
- * @mm: process address space
61
- * @mn: MMU notifier structure
62
- * @type: type of MMU notifier
63
- * @work: destruction work item
64
- * @node: hash table node to find structure by adev and mn
65
- * @lock: rw semaphore protecting the notifier nodes
66
- * @objects: interval tree containing amdgpu_mn_nodes
67
- * @read_lock: mutex for recursive locking of @lock
68
- * @recursion: depth of recursion
69
- *
70
- * Data for each amdgpu device and process address space.
71
- */
72
-struct amdgpu_mn {
73
-	/* constant after initialisation */
74
-	struct amdgpu_device	*adev;
75
-	struct mm_struct	*mm;
76
-	struct mmu_notifier	mn;
77
-	enum amdgpu_mn_type	type;
78
-
79
-	/* only used on destruction */
80
-	struct work_struct	work;
81
-
82
-	/* protected by adev->mn_lock */
83
-	struct hlist_node	node;
84
-
85
-	/* objects protected by lock */
86
-	struct rw_semaphore	lock;
87
-	struct rb_root_cached	objects;
88
-	struct mutex		read_lock;
89
-	atomic_t		recursion;
90
-};
91
-
92
-/**
93
- * struct amdgpu_mn_node
94
- *
95
- * @it: interval node defining start-last of the affected address range
96
- * @bos: list of all BOs in the affected address range
97
- *
98
- * Manages all BOs which are affected of a certain range of address space.
99
- */
100
-struct amdgpu_mn_node {
101
-	struct interval_tree_node	it;
102
-	struct list_head		bos;
103
-};
104
-
105
-/**
106
- * amdgpu_mn_destroy - destroy the MMU notifier
107
- *
108
- * @work: previously sheduled work item
109
- *
110
- * Lazy destroys the notifier from a work item
111
- */
112
-static void amdgpu_mn_destroy(struct work_struct *work)
113
-{
114
-	struct amdgpu_mn *amn = container_of(work, struct amdgpu_mn, work);
115
-	struct amdgpu_device *adev = amn->adev;
116
-	struct amdgpu_mn_node *node, *next_node;
117
-	struct amdgpu_bo *bo, *next_bo;
118
-
119
-	mutex_lock(&adev->mn_lock);
120
-	down_write(&amn->lock);
121
-	hash_del(&amn->node);
122
-	rbtree_postorder_for_each_entry_safe(node, next_node,
123
-					     &amn->objects.rb_root, it.rb) {
124
-		list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
125
-			bo->mn = NULL;
126
-			list_del_init(&bo->mn_list);
127
-		}
128
-		kfree(node);
129
-	}
130
-	up_write(&amn->lock);
131
-	mutex_unlock(&adev->mn_lock);
132
-	mmu_notifier_unregister_no_release(&amn->mn, amn->mm);
133
-	kfree(amn);
134
-}
135
-
136
-/**
137
- * amdgpu_mn_release - callback to notify about mm destruction
138
- *
139
- * @mn: our notifier
140
- * @mm: the mm this callback is about
141
- *
142
- * Shedule a work item to lazy destroy our notifier.
143
- */
144
-static void amdgpu_mn_release(struct mmu_notifier *mn,
145
-			      struct mm_struct *mm)
146
-{
147
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
148
-
149
-	INIT_WORK(&amn->work, amdgpu_mn_destroy);
150
-	schedule_work(&amn->work);
151
-}
152
-
153
-
154
-/**
155
- * amdgpu_mn_lock - take the write side lock for this notifier
156
- *
157
- * @mn: our notifier
158
- */
159
-void amdgpu_mn_lock(struct amdgpu_mn *mn)
160
-{
161
-	if (mn)
162
-		down_write(&mn->lock);
163
-}
164
-
165
-/**
166
- * amdgpu_mn_unlock - drop the write side lock for this notifier
167
- *
168
- * @mn: our notifier
169
- */
170
-void amdgpu_mn_unlock(struct amdgpu_mn *mn)
171
-{
172
-	if (mn)
173
-		up_write(&mn->lock);
174
-}
175
-
176
-/**
177
- * amdgpu_mn_read_lock - take the read side lock for this notifier
178
- *
179
- * @amn: our notifier
180
- */
181
-static int amdgpu_mn_read_lock(struct amdgpu_mn *amn, bool blockable)
182
-{
183
-	if (blockable)
184
-		mutex_lock(&amn->read_lock);
185
-	else if (!mutex_trylock(&amn->read_lock))
186
-		return -EAGAIN;
187
-
188
-	if (atomic_inc_return(&amn->recursion) == 1)
189
-		down_read_non_owner(&amn->lock);
190
-	mutex_unlock(&amn->read_lock);
191
-
192
-	return 0;
193
-}
194
-
195
-/**
196
- * amdgpu_mn_read_unlock - drop the read side lock for this notifier
197
- *
198
- * @amn: our notifier
199
- */
200
-static void amdgpu_mn_read_unlock(struct amdgpu_mn *amn)
201
-{
202
-	if (atomic_dec_return(&amn->recursion) == 0)
203
-		up_read_non_owner(&amn->lock);
204
-}
205
-
206
-/**
207
- * amdgpu_mn_invalidate_node - unmap all BOs of a node
208
- *
209
- * @node: the node with the BOs to unmap
210
- * @start: start of address range affected
211
- * @end: end of address range affected
56
+ * @mni: the range (mm) is about to update
57
+ * @range: details on the invalidation
58
+ * @cur_seq: Value to pass to mmu_interval_set_seq()
21259  *
21360  * Block for operations on BOs to finish and mark pages as accessed and
21461  * potentially dirty.
21562  */
216
-static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
217
-				      unsigned long start,
218
-				      unsigned long end)
63
+static bool amdgpu_mn_invalidate_gfx(struct mmu_interval_notifier *mni,
64
+				     const struct mmu_notifier_range *range,
65
+				     unsigned long cur_seq)
21966 {
220
-	struct amdgpu_bo *bo;
67
+	struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier);
68
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
22169 	long r;
22270 
223
-	list_for_each_entry(bo, &node->bos, mn_list) {
71
+	if (!mmu_notifier_range_blockable(range))
72
+		return false;
22473 
225
-		if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
226
-			continue;
74
+	mutex_lock(&adev->notifier_lock);
22775 
228
-		r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
229
-			true, false, MAX_SCHEDULE_TIMEOUT);
230
-		if (r <= 0)
231
-			DRM_ERROR("(%ld) failed to wait for user bo\n", r);
76
+	mmu_interval_set_seq(mni, cur_seq);
23277 
233
-		amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
234
-	}
78
+	r = dma_resv_wait_timeout_rcu(bo->tbo.base.resv, true, false,
79
+				      MAX_SCHEDULE_TIMEOUT);
80
+	mutex_unlock(&adev->notifier_lock);
81
+	if (r <= 0)
82
+		DRM_ERROR("(%ld) failed to wait for user bo\n", r);
83
+	return true;
23584 }
23685 
237
-/**
238
- * amdgpu_mn_invalidate_range_start_gfx - callback to notify about mm change
239
- *
240
- * @mn: our notifier
241
- * @mm: the mm this callback is about
242
- * @start: start of updated range
243
- * @end: end of updated range
244
- *
245
- * Block for operations on BOs to finish and mark pages as accessed and
246
- * potentially dirty.
247
- */
248
-static int amdgpu_mn_invalidate_range_start_gfx(struct mmu_notifier *mn,
249
-						 struct mm_struct *mm,
250
-						 unsigned long start,
251
-						 unsigned long end,
252
-						 bool blockable)
253
-{
254
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
255
-	struct interval_tree_node *it;
256
-
257
-	/* notification is exclusive, but interval is inclusive */
258
-	end -= 1;
259
-
260
-	/* TODO we should be able to split locking for interval tree and
261
-	 * amdgpu_mn_invalidate_node
262
-	 */
263
-	if (amdgpu_mn_read_lock(amn, blockable))
264
-		return -EAGAIN;
265
-
266
-	it = interval_tree_iter_first(&amn->objects, start, end);
267
-	while (it) {
268
-		struct amdgpu_mn_node *node;
269
-
270
-		if (!blockable) {
271
-			amdgpu_mn_read_unlock(amn);
272
-			return -EAGAIN;
273
-		}
274
-
275
-		node = container_of(it, struct amdgpu_mn_node, it);
276
-		it = interval_tree_iter_next(it, start, end);
277
-
278
-		amdgpu_mn_invalidate_node(node, start, end);
279
-	}
280
-
281
-	return 0;
282
-}
283
-
284
-/**
285
- * amdgpu_mn_invalidate_range_start_hsa - callback to notify about mm change
286
- *
287
- * @mn: our notifier
288
- * @mm: the mm this callback is about
289
- * @start: start of updated range
290
- * @end: end of updated range
291
- *
292
- * We temporarily evict all BOs between start and end. This
293
- * necessitates evicting all user-mode queues of the process. The BOs
294
- * are restorted in amdgpu_mn_invalidate_range_end_hsa.
295
- */
296
-static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn,
297
-						 struct mm_struct *mm,
298
-						 unsigned long start,
299
-						 unsigned long end,
300
-						 bool blockable)
301
-{
302
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
303
-	struct interval_tree_node *it;
304
-
305
-	/* notification is exclusive, but interval is inclusive */
306
-	end -= 1;
307
-
308
-	if (amdgpu_mn_read_lock(amn, blockable))
309
-		return -EAGAIN;
310
-
311
-	it = interval_tree_iter_first(&amn->objects, start, end);
312
-	while (it) {
313
-		struct amdgpu_mn_node *node;
314
-		struct amdgpu_bo *bo;
315
-
316
-		if (!blockable) {
317
-			amdgpu_mn_read_unlock(amn);
318
-			return -EAGAIN;
319
-		}
320
-
321
-		node = container_of(it, struct amdgpu_mn_node, it);
322
-		it = interval_tree_iter_next(it, start, end);
323
-
324
-		list_for_each_entry(bo, &node->bos, mn_list) {
325
-			struct kgd_mem *mem = bo->kfd_bo;
326
-
327
-			if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm,
328
-							 start, end))
329
-				amdgpu_amdkfd_evict_userptr(mem, mm);
330
-		}
331
-	}
332
-
333
-	return 0;
334
-}
335
-
336
-/**
337
- * amdgpu_mn_invalidate_range_end - callback to notify about mm change
338
- *
339
- * @mn: our notifier
340
- * @mm: the mm this callback is about
341
- * @start: start of updated range
342
- * @end: end of updated range
343
- *
344
- * Release the lock again to allow new command submissions.
345
- */
346
-static void amdgpu_mn_invalidate_range_end(struct mmu_notifier *mn,
347
-					   struct mm_struct *mm,
348
-					   unsigned long start,
349
-					   unsigned long end)
350
-{
351
-	struct amdgpu_mn *amn = container_of(mn, struct amdgpu_mn, mn);
352
-
353
-	amdgpu_mn_read_unlock(amn);
354
-}
355
-
356
-static const struct mmu_notifier_ops amdgpu_mn_ops[] = {
357
-	[AMDGPU_MN_TYPE_GFX] = {
358
-		.release = amdgpu_mn_release,
359
-		.invalidate_range_start = amdgpu_mn_invalidate_range_start_gfx,
360
-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
361
-	},
362
-	[AMDGPU_MN_TYPE_HSA] = {
363
-		.release = amdgpu_mn_release,
364
-		.invalidate_range_start = amdgpu_mn_invalidate_range_start_hsa,
365
-		.invalidate_range_end = amdgpu_mn_invalidate_range_end,
366
-	},
86
+static const struct mmu_interval_notifier_ops amdgpu_mn_gfx_ops = {
87
+	.invalidate = amdgpu_mn_invalidate_gfx,
36788 };
36889 
369
-/* Low bits of any reasonable mm pointer will be unused due to struct
370
- * alignment. Use these bits to make a unique key from the mm pointer
371
- * and notifier type.
372
- */
373
-#define AMDGPU_MN_KEY(mm, type) ((unsigned long)(mm) + (type))
374
-
37590 /**
376
- * amdgpu_mn_get - create notifier context
91
+ * amdgpu_mn_invalidate_hsa - callback to notify about mm change
37792  *
378
- * @adev: amdgpu device pointer
379
- * @type: type of MMU notifier context
93
+ * @mni: the range (mm) is about to update
94
+ * @range: details on the invalidation
95
+ * @cur_seq: Value to pass to mmu_interval_set_seq()
38096  *
381
- * Creates a notifier context for current->mm.
97
+ * We temporarily evict the BO attached to this range. This necessitates
98
+ * evicting all user-mode queues of the process.
38299  */
383
-struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev,
384
-				enum amdgpu_mn_type type)
100
+static bool amdgpu_mn_invalidate_hsa(struct mmu_interval_notifier *mni,
101
+				     const struct mmu_notifier_range *range,
102
+				     unsigned long cur_seq)
385103 {
386
-	struct mm_struct *mm = current->mm;
387
-	struct amdgpu_mn *amn;
388
-	unsigned long key = AMDGPU_MN_KEY(mm, type);
389
-	int r;
104
+	struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier);
105
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
390106 
391
-	mutex_lock(&adev->mn_lock);
392
-	if (down_write_killable(&mm->mmap_sem)) {
393
-		mutex_unlock(&adev->mn_lock);
394
-		return ERR_PTR(-EINTR);
395
-	}
107
+	if (!mmu_notifier_range_blockable(range))
108
+		return false;
396109 
397
-	hash_for_each_possible(adev->mn_hash, amn, node, key)
398
-		if (AMDGPU_MN_KEY(amn->mm, amn->type) == key)
399
-			goto release_locks;
110
+	mutex_lock(&adev->notifier_lock);
400111 
401
-	amn = kzalloc(sizeof(*amn), GFP_KERNEL);
402
-	if (!amn) {
403
-		amn = ERR_PTR(-ENOMEM);
404
-		goto release_locks;
405
-	}
112
+	mmu_interval_set_seq(mni, cur_seq);
406113 
407
-	amn->adev = adev;
408
-	amn->mm = mm;
409
-	init_rwsem(&amn->lock);
410
-	amn->type = type;
411
-	amn->mn.ops = &amdgpu_mn_ops[type];
412
-	amn->objects = RB_ROOT_CACHED;
413
-	mutex_init(&amn->read_lock);
414
-	atomic_set(&amn->recursion, 0);
114
+	amdgpu_amdkfd_evict_userptr(bo->kfd_bo, bo->notifier.mm);
115
+	mutex_unlock(&adev->notifier_lock);
415116 
416
-	r = __mmu_notifier_register(&amn->mn, mm);
417
-	if (r)
418
-		goto free_amn;
419
-
420
-	hash_add(adev->mn_hash, &amn->node, AMDGPU_MN_KEY(mm, type));
421
-
422
-release_locks:
423
-	up_write(&mm->mmap_sem);
424
-	mutex_unlock(&adev->mn_lock);
425
-
426
-	return amn;
427
-
428
-free_amn:
429
-	up_write(&mm->mmap_sem);
430
-	mutex_unlock(&adev->mn_lock);
431
-	kfree(amn);
432
-
433
-	return ERR_PTR(r);
117
+	return true;
434118 }
119
+
120
+static const struct mmu_interval_notifier_ops amdgpu_mn_hsa_ops = {
121
+	.invalidate = amdgpu_mn_invalidate_hsa,
122
+};
435123 
436124 /**
437125  * amdgpu_mn_register - register a BO for notifier updates
....@@ -439,59 +127,18 @@
439127  * @bo: amdgpu buffer object
440128  * @addr: userptr addr we should monitor
441129  *
442
- * Registers an MMU notifier for the given BO at the specified address.
130
+ * Registers a mmu_notifier for the given BO at the specified address.
443131  * Returns 0 on success, -ERRNO if anything goes wrong.
444132  */
445133 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
446134 {
447
-	unsigned long end = addr + amdgpu_bo_size(bo) - 1;
448
-	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
449
-	enum amdgpu_mn_type type =
450
-		bo->kfd_bo ? AMDGPU_MN_TYPE_HSA : AMDGPU_MN_TYPE_GFX;
451
-	struct amdgpu_mn *amn;
452
-	struct amdgpu_mn_node *node = NULL, *new_node;
453
-	struct list_head bos;
454
-	struct interval_tree_node *it;
455
-
456
-	amn = amdgpu_mn_get(adev, type);
457
-	if (IS_ERR(amn))
458
-		return PTR_ERR(amn);
459
-
460
-	new_node = kmalloc(sizeof(*new_node), GFP_KERNEL);
461
-	if (!new_node)
462
-		return -ENOMEM;
463
-
464
-	INIT_LIST_HEAD(&bos);
465
-
466
-	down_write(&amn->lock);
467
-
468
-	while ((it = interval_tree_iter_first(&amn->objects, addr, end))) {
469
-		kfree(node);
470
-		node = container_of(it, struct amdgpu_mn_node, it);
471
-		interval_tree_remove(&node->it, &amn->objects);
472
-		addr = min(it->start, addr);
473
-		end = max(it->last, end);
474
-		list_splice(&node->bos, &bos);
475
-	}
476
-
477
-	if (!node)
478
-		node = new_node;
479
-	else
480
-		kfree(new_node);
481
-
482
-	bo->mn = amn;
483
-
484
-	node->it.start = addr;
485
-	node->it.last = end;
486
-	INIT_LIST_HEAD(&node->bos);
487
-	list_splice(&bos, &node->bos);
488
-	list_add(&bo->mn_list, &node->bos);
489
-
490
-	interval_tree_insert(&node->it, &amn->objects);
491
-
492
-	up_write(&amn->lock);
493
-
494
-	return 0;
135
+	if (bo->kfd_bo)
136
+		return mmu_interval_notifier_insert(&bo->notifier, current->mm,
137
+						    addr, amdgpu_bo_size(bo),
138
+						    &amdgpu_mn_hsa_ops);
139
+	return mmu_interval_notifier_insert(&bo->notifier, current->mm, addr,
140
+					    amdgpu_bo_size(bo),
141
+					    &amdgpu_mn_gfx_ops);
495142 }
496143 
497144 /**
....@@ -499,39 +146,12 @@
499146  *
500147  * @bo: amdgpu buffer object
501148  *
502
- * Remove any registration of MMU notifier updates from the buffer object.
149
+ * Remove any registration of mmu notifier updates from the buffer object.
503150  */
504151 void amdgpu_mn_unregister(struct amdgpu_bo *bo)
505152 {
506
-	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
507
-	struct amdgpu_mn *amn;
508
-	struct list_head *head;
509
-
510
-	mutex_lock(&adev->mn_lock);
511
-
512
-	amn = bo->mn;
513
-	if (amn == NULL) {
514
-		mutex_unlock(&adev->mn_lock);
153
+	if (!bo->notifier.mm)
515154 		return;
516
-	}
517
-
518
-	down_write(&amn->lock);
519
-
520
-	/* save the next list entry for later */
521
-	head = bo->mn_list.next;
522
-
523
-	bo->mn = NULL;
524
-	list_del_init(&bo->mn_list);
525
-
526
-	if (list_empty(head)) {
527
-		struct amdgpu_mn_node *node;
528
-
529
-		node = container_of(head, struct amdgpu_mn_node, bos);
530
-		interval_tree_remove(&node->it, &amn->objects);
531
-		kfree(node);
532
-	}
533
-
534
-	up_write(&amn->lock);
535
-	mutex_unlock(&adev->mn_lock);
155
+	mmu_interval_notifier_remove(&bo->notifier);
156
+	bo->notifier.mm = NULL;
536157 }
537
-