add dts config

hc

2023-12-06 08f87f769b595151be1afeff53e144f543faa614

 kernel/drivers/gpu/drm/ttm/ttm_bo_util.c
..
..
@@ -38,7 +38,7 @@
38
38
 #include <linux/slab.h>
39
39
 #include <linux/vmalloc.h>
40
40
 #include <linux/module.h>
41
-#include <linux/reservation.h>
41
+#include <linux/dma-resv.h>
42
42
 
43
43
 struct ttm_transfer_obj {
44
44
 	struct ttm_buffer_object base;
..
..
@@ -47,15 +47,15 @@
47
47
 
48
48
 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
49
49
 {
50
-	ttm_bo_mem_put(bo, &bo->mem);
50
+	ttm_resource_free(bo, &bo->mem);
51
51
 }
52
52
 
53
53
 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
54
54
 		   struct ttm_operation_ctx *ctx,
55
-		    struct ttm_mem_reg *new_mem)
55
+		    struct ttm_resource *new_mem)
56
56
 {
57
57
 	struct ttm_tt *ttm = bo->ttm;
58
-	struct ttm_mem_reg *old_mem = &bo->mem;
58
+	struct ttm_resource *old_mem = &bo->mem;
59
59
 	int ret;
60
60
 
61
61
 	if (old_mem->mem_type != TTM_PL_SYSTEM) {
..
..
@@ -67,10 +67,8 @@
67
67
 			return ret;
68
68
 		}
69
69
 
70
-		ttm_tt_unbind(ttm);
70
+		ttm_bo_tt_unbind(bo);
71
71
 		ttm_bo_free_old_node(bo);
72
-		ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
73
-				TTM_PL_MASK_MEM);
74
72
 		old_mem->mem_type = TTM_PL_SYSTEM;
75
73
 	}
76
74
 
..
..
@@ -79,154 +77,70 @@
79
77
 		return ret;
80
78
 
81
79
 	if (new_mem->mem_type != TTM_PL_SYSTEM) {
82
-		ret = ttm_tt_bind(ttm, new_mem, ctx);
80
+
81
+		ret = ttm_tt_populate(bo->bdev, ttm, ctx);
82
+		if (unlikely(ret != 0))
83
+			return ret;
84
+
85
+		ret = ttm_bo_tt_bind(bo, new_mem);
83
86
 		if (unlikely(ret != 0))
84
87
 			return ret;
85
88
 	}
86
89
 
87
-	*old_mem = *new_mem;
88
-	new_mem->mm_node = NULL;
89
-
90
+	ttm_bo_assign_mem(bo, new_mem);
90
91
 	return 0;
91
92
 }
92
93
 EXPORT_SYMBOL(ttm_bo_move_ttm);
93
94
 
94
-int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
95
+int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
96
+		       struct ttm_resource *mem)
95
97
 {
96
-	if (likely(man->io_reserve_fastpath))
98
+	if (mem->bus.offset || mem->bus.addr)
97
99
 		return 0;
98
100
 
99
-	if (interruptible)
100
-		return mutex_lock_interruptible(&man->io_reserve_mutex);
101
-
102
-	mutex_lock(&man->io_reserve_mutex);
103
-	return 0;
104
-}
105
-EXPORT_SYMBOL(ttm_mem_io_lock);
106
-
107
-void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
108
-{
109
-	if (likely(man->io_reserve_fastpath))
110
-		return;
111
-
112
-	mutex_unlock(&man->io_reserve_mutex);
113
-}
114
-EXPORT_SYMBOL(ttm_mem_io_unlock);
115
-
116
-static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
117
-{
118
-	struct ttm_buffer_object *bo;
119
-
120
-	if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
121
-		return -EAGAIN;
122
-
123
-	bo = list_first_entry(&man->io_reserve_lru,
124
-			      struct ttm_buffer_object,
125
-			      io_reserve_lru);
126
-	list_del_init(&bo->io_reserve_lru);
127
-	ttm_bo_unmap_virtual_locked(bo);
128
-
129
-	return 0;
130
-}
131
-
132
-
133
-int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
134
-		       struct ttm_mem_reg *mem)
135
-{
136
-	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
137
-	int ret = 0;
138
-
101
+	mem->bus.is_iomem = false;
139
102
 	if (!bdev->driver->io_mem_reserve)
140
103
 		return 0;
141
-	if (likely(man->io_reserve_fastpath))
142
-		return bdev->driver->io_mem_reserve(bdev, mem);
143
104
 
144
-	if (bdev->driver->io_mem_reserve &&
145
-	    mem->bus.io_reserved_count++ == 0) {
146
-retry:
147
-		ret = bdev->driver->io_mem_reserve(bdev, mem);
148
-		if (ret == -EAGAIN) {
149
-			ret = ttm_mem_io_evict(man);
150
-			if (ret == 0)
151
-				goto retry;
152
-		}
153
-	}
154
-	return ret;
105
+	return bdev->driver->io_mem_reserve(bdev, mem);
155
106
 }
156
-EXPORT_SYMBOL(ttm_mem_io_reserve);
157
107
 
158
108
 void ttm_mem_io_free(struct ttm_bo_device *bdev,
159
-		     struct ttm_mem_reg *mem)
109
+		     struct ttm_resource *mem)
160
110
 {
161
-	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
162
-
163
-	if (likely(man->io_reserve_fastpath))
111
+	if (!mem->bus.offset && !mem->bus.addr)
164
112
 		return;
165
113
 
166
-	if (bdev->driver->io_mem_reserve &&
167
-	    --mem->bus.io_reserved_count == 0 &&
168
-	    bdev->driver->io_mem_free)
114
+	if (bdev->driver->io_mem_free)
169
115
 		bdev->driver->io_mem_free(bdev, mem);
170
116
 
171
-}
172
-EXPORT_SYMBOL(ttm_mem_io_free);
173
-
174
-int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
175
-{
176
-	struct ttm_mem_reg *mem = &bo->mem;
177
-	int ret;
178
-
179
-	if (!mem->bus.io_reserved_vm) {
180
-		struct ttm_mem_type_manager *man =
181
-			&bo->bdev->man[mem->mem_type];
182
-
183
-		ret = ttm_mem_io_reserve(bo->bdev, mem);
184
-		if (unlikely(ret != 0))
185
-			return ret;
186
-		mem->bus.io_reserved_vm = true;
187
-		if (man->use_io_reserve_lru)
188
-			list_add_tail(&bo->io_reserve_lru,
189
-				      &man->io_reserve_lru);
190
-	}
191
-	return 0;
117
+	mem->bus.offset = 0;
118
+	mem->bus.addr = NULL;
192
119
 }
193
120
 
194
-void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
121
+static int ttm_resource_ioremap(struct ttm_bo_device *bdev,
122
+			       struct ttm_resource *mem,
123
+			       void **virtual)
195
124
 {
196
-	struct ttm_mem_reg *mem = &bo->mem;
197
-
198
-	if (mem->bus.io_reserved_vm) {
199
-		mem->bus.io_reserved_vm = false;
200
-		list_del_init(&bo->io_reserve_lru);
201
-		ttm_mem_io_free(bo->bdev, mem);
202
-	}
203
-}
204
-
205
-static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
206
-			void **virtual)
207
-{
208
-	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
209
125
 	int ret;
210
126
 	void *addr;
211
127
 
212
128
 	*virtual = NULL;
213
-	(void) ttm_mem_io_lock(man, false);
214
129
 	ret = ttm_mem_io_reserve(bdev, mem);
215
-	ttm_mem_io_unlock(man);
216
130
 	if (ret || !mem->bus.is_iomem)
217
131
 		return ret;
218
132
 
219
133
 	if (mem->bus.addr) {
220
134
 		addr = mem->bus.addr;
221
135
 	} else {
136
+		size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
137
+
222
138
 		if (mem->placement & TTM_PL_FLAG_WC)
223
-			addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
139
+			addr = ioremap_wc(mem->bus.offset, bus_size);
224
140
 		else
225
-			addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
141
+			addr = ioremap(mem->bus.offset, bus_size);
226
142
 		if (!addr) {
227
-			(void) ttm_mem_io_lock(man, false);
228
143
 			ttm_mem_io_free(bdev, mem);
229
-			ttm_mem_io_unlock(man);
230
144
 			return -ENOMEM;
231
145
 		}
232
146
 	}
..
..
@@ -234,18 +148,13 @@
234
148
 	return 0;
235
149
 }
236
150
 
237
-static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
238
-			 void *virtual)
151
+static void ttm_resource_iounmap(struct ttm_bo_device *bdev,
152
+				struct ttm_resource *mem,
153
+				void *virtual)
239
154
 {
240
-	struct ttm_mem_type_manager *man;
241
-
242
-	man = &bdev->man[mem->mem_type];
243
-
244
155
 	if (virtual && mem->bus.addr == NULL)
245
156
 		iounmap(virtual);
246
-	(void) ttm_mem_io_lock(man, false);
247
157
 	ttm_mem_io_free(bdev, mem);
248
-	ttm_mem_io_unlock(man);
249
158
 }
250
159
 
251
160
 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
..
..
@@ -261,54 +170,6 @@
261
170
 	return 0;
262
171
 }
263
172
 
264
-#ifdef CONFIG_X86
265
-#define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
266
-#define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
267
-#else
268
-#define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
269
-#define __ttm_kunmap_atomic(__addr) vunmap(__addr)
270
-#endif
271
-
272
-
273
-/**
274
- * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
275
- * specified page protection.
276
- *
277
- * @page: The page to map.
278
- * @prot: The page protection.
279
- *
280
- * This function maps a TTM page using the kmap_atomic api if available,
281
- * otherwise falls back to vmap. The user must make sure that the
282
- * specified page does not have an aliased mapping with a different caching
283
- * policy unless the architecture explicitly allows it. Also mapping and
284
- * unmapping using this api must be correctly nested. Unmapping should
285
- * occur in the reverse order of mapping.
286
- */
287
-void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
288
-{
289
-	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
290
-		return kmap_atomic(page);
291
-	else
292
-		return __ttm_kmap_atomic_prot(page, prot);
293
-}
294
-EXPORT_SYMBOL(ttm_kmap_atomic_prot);
295
-
296
-/**
297
- * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
298
- * ttm_kmap_atomic_prot.
299
- *
300
- * @addr: The virtual address from the map.
301
- * @prot: The page protection.
302
- */
303
-void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
304
-{
305
-	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
306
-		kunmap_atomic(addr);
307
-	else
308
-		__ttm_kunmap_atomic(addr);
309
-}
310
-EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
311
-
312
173
 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
313
174
 				unsigned long page,
314
175
 				pgprot_t prot)
..
..
@@ -320,13 +181,15 @@
320
181
 		return -ENOMEM;
321
182
 
322
183
 	src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
323
-	dst = ttm_kmap_atomic_prot(d, prot);
324
-	if (!dst)
325
-		return -ENOMEM;
184
+	/*
185
+	 * Ensure that a highmem page is mapped with the correct
186
+	 * pgprot. For non highmem the mapping is already there.
187
+	 */
188
+	dst = kmap_local_page_prot(d, prot);
326
189
 
327
190
 	memcpy_fromio(dst, src, PAGE_SIZE);
328
191
 
329
-	ttm_kunmap_atomic_prot(dst, prot);
192
+	kunmap_local(dst);
330
193
 
331
194
 	return 0;
332
195
 }
..
..
@@ -342,26 +205,28 @@
342
205
 		return -ENOMEM;
343
206
 
344
207
 	dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
345
-	src = ttm_kmap_atomic_prot(s, prot);
346
-	if (!src)
347
-		return -ENOMEM;
208
+	/*
209
+	 * Ensure that a highmem page is mapped with the correct
210
+	 * pgprot. For non highmem the mapping is already there.
211
+	 */
212
+	src = kmap_local_page_prot(s, prot);
348
213
 
349
214
 	memcpy_toio(dst, src, PAGE_SIZE);
350
215
 
351
-	ttm_kunmap_atomic_prot(src, prot);
216
+	kunmap_local(src);
352
217
 
353
218
 	return 0;
354
219
 }
355
220
 
356
221
 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
357
222
 		       struct ttm_operation_ctx *ctx,
358
-		       struct ttm_mem_reg *new_mem)
223
+		       struct ttm_resource *new_mem)
359
224
 {
360
225
 	struct ttm_bo_device *bdev = bo->bdev;
361
-	struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
226
+	struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
362
227
 	struct ttm_tt *ttm = bo->ttm;
363
-	struct ttm_mem_reg *old_mem = &bo->mem;
364
-	struct ttm_mem_reg old_copy = *old_mem;
228
+	struct ttm_resource *old_mem = &bo->mem;
229
+	struct ttm_resource old_copy = *old_mem;
365
230
 	void *old_iomap;
366
231
 	void *new_iomap;
367
232
 	int ret;
..
..
@@ -374,10 +239,10 @@
374
239
 	if (ret)
375
240
 		return ret;
376
241
 
377
-	ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
242
+	ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap);
378
243
 	if (ret)
379
244
 		return ret;
380
-	ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
245
+	ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap);
381
246
 	if (ret)
382
247
 		goto out;
383
248
 
..
..
@@ -391,7 +256,7 @@
391
256
 	 * Don't move nonexistent data. Clear destination instead.
392
257
 	 */
393
258
 	if (old_iomap == NULL &&
394
-	    (ttm == NULL || (ttm->state == tt_unpopulated &&
259
+	    (ttm == NULL || (!ttm_tt_is_populated(ttm) &&
395
260
 			     !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
396
261
 		memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
397
262
 		goto out2;
..
..
@@ -401,7 +266,7 @@
401
266
 	 * TTM might be null for moves within the same region.
402
267
 	 */
403
268
 	if (ttm) {
404
-		ret = ttm_tt_populate(ttm, ctx);
269
+		ret = ttm_tt_populate(bdev, ttm, ctx);
405
270
 		if (ret)
406
271
 			goto out1;
407
272
 	}
..
..
@@ -436,24 +301,22 @@
436
301
 	mb();
437
302
 out2:
438
303
 	old_copy = *old_mem;
439
-	*old_mem = *new_mem;
440
-	new_mem->mm_node = NULL;
441
304
 
442
-	if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
443
-		ttm_tt_destroy(ttm);
444
-		bo->ttm = NULL;
445
-	}
305
+	ttm_bo_assign_mem(bo, new_mem);
306
+
307
+	if (!man->use_tt)
308
+		ttm_bo_tt_destroy(bo);
446
309
 
447
310
 out1:
448
-	ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
311
+	ttm_resource_iounmap(bdev, old_mem, new_iomap);
449
312
 out:
450
-	ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
313
+	ttm_resource_iounmap(bdev, &old_copy, old_iomap);
451
314
 
452
315
 	/*
453
316
 	 * On error, keep the mm node!
454
317
 	 */
455
318
 	if (!ret)
456
-		ttm_bo_mem_put(bo, &old_copy);
319
+		ttm_resource_free(bo, &old_copy);
457
320
 	return ret;
458
321
 }
459
322
 EXPORT_SYMBOL(ttm_bo_move_memcpy);
..
..
@@ -503,23 +366,22 @@
503
366
 	 * TODO: Explicit member copy would probably be better here.
504
367
 	 */
505
368
 
506
-	atomic_inc(&bo->bdev->glob->bo_count);
369
+	atomic_inc(&ttm_bo_glob.bo_count);
507
370
 	INIT_LIST_HEAD(&fbo->base.ddestroy);
508
371
 	INIT_LIST_HEAD(&fbo->base.lru);
509
372
 	INIT_LIST_HEAD(&fbo->base.swap);
510
-	INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
511
-	mutex_init(&fbo->base.wu_mutex);
512
373
 	fbo->base.moving = NULL;
513
-	drm_vma_node_reset(&fbo->base.vma_node);
514
-	atomic_set(&fbo->base.cpu_writers, 0);
374
+	drm_vma_node_reset(&fbo->base.base.vma_node);
515
375
 
516
-	kref_init(&fbo->base.list_kref);
517
376
 	kref_init(&fbo->base.kref);
518
377
 	fbo->base.destroy = &ttm_transfered_destroy;
519
378
 	fbo->base.acc_size = 0;
520
-	fbo->base.resv = &fbo->base.ttm_resv;
521
-	reservation_object_init(fbo->base.resv);
522
-	ret = reservation_object_trylock(fbo->base.resv);
379
+	if (bo->type != ttm_bo_type_sg)
380
+		fbo->base.base.resv = &fbo->base.base._resv;
381
+
382
+	dma_resv_init(&fbo->base.base._resv);
383
+	fbo->base.base.dev = NULL;
384
+	ret = dma_resv_trylock(&fbo->base.base._resv);
523
385
 	WARN_ON(!ret);
524
386
 
525
387
 	*new_obj = &fbo->base;
..
..
@@ -539,13 +401,13 @@
539
401
 		tmp = pgprot_noncached(tmp);
540
402
 #endif
541
403
 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
542
-    defined(__powerpc__)
404
+    defined(__powerpc__) || defined(__mips__)
543
405
 	if (caching_flags & TTM_PL_FLAG_WC)
544
406
 		tmp = pgprot_writecombine(tmp);
545
407
 	else
546
408
 		tmp = pgprot_noncached(tmp);
547
409
 #endif
548
-#if defined(__sparc__) || defined(__mips__)
410
+#if defined(__sparc__)
549
411
 	tmp = pgprot_noncached(tmp);
550
412
 #endif
551
413
 	return tmp;
..
..
@@ -557,7 +419,7 @@
557
419
 			  unsigned long size,
558
420
 			  struct ttm_bo_kmap_obj *map)
559
421
 {
560
-	struct ttm_mem_reg *mem = &bo->mem;
422
+	struct ttm_resource *mem = &bo->mem;
561
423
 
562
424
 	if (bo->mem.bus.addr) {
563
425
 		map->bo_kmap_type = ttm_bo_map_premapped;
..
..
@@ -565,11 +427,11 @@
565
427
 	} else {
566
428
 		map->bo_kmap_type = ttm_bo_map_iomap;
567
429
 		if (mem->placement & TTM_PL_FLAG_WC)
568
-			map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
430
+			map->virtual = ioremap_wc(bo->mem.bus.offset + offset,
569
431
 						  size);
570
432
 		else
571
-			map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
572
-						       size);
433
+			map->virtual = ioremap(bo->mem.bus.offset + offset,
434
+					       size);
573
435
 	}
574
436
 	return (!map->virtual) ? -ENOMEM : 0;
575
437
 }
..
..
@@ -579,7 +441,7 @@
579
441
 			   unsigned long num_pages,
580
442
 			   struct ttm_bo_kmap_obj *map)
581
443
 {
582
-	struct ttm_mem_reg *mem = &bo->mem;
444
+	struct ttm_resource *mem = &bo->mem;
583
445
 	struct ttm_operation_ctx ctx = {
584
446
 		.interruptible = false,
585
447
 		.no_wait_gpu = false
..
..
@@ -590,7 +452,7 @@
590
452
 
591
453
 	BUG_ON(!ttm);
592
454
 
593
-	ret = ttm_tt_populate(ttm, &ctx);
455
+	ret = ttm_tt_populate(bo->bdev, ttm, &ctx);
594
456
 	if (ret)
595
457
 		return ret;
596
458
 
..
..
@@ -620,8 +482,6 @@
620
482
 		unsigned long start_page, unsigned long num_pages,
621
483
 		struct ttm_bo_kmap_obj *map)
622
484
 {
623
-	struct ttm_mem_type_manager *man =
624
-		&bo->bdev->man[bo->mem.mem_type];
625
485
 	unsigned long offset, size;
626
486
 	int ret;
627
487
 
..
..
@@ -631,13 +491,8 @@
631
491
 		return -EINVAL;
632
492
 	if (start_page > bo->num_pages)
633
493
 		return -EINVAL;
634
-#if 0
635
-	if (num_pages > 1 && !capable(CAP_SYS_ADMIN))
636
-		return -EPERM;
637
-#endif
638
-	(void) ttm_mem_io_lock(man, false);
494
+
639
495
 	ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
640
-	ttm_mem_io_unlock(man);
641
496
 	if (ret)
642
497
 		return ret;
643
498
 	if (!bo->mem.bus.is_iomem) {
..
..
@@ -652,10 +507,6 @@
652
507
 
653
508
 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
654
509
 {
655
-	struct ttm_buffer_object *bo = map->bo;
656
-	struct ttm_mem_type_manager *man =
657
-		&bo->bdev->man[bo->mem.mem_type];
658
-
659
510
 	if (!map->virtual)
660
511
 		return;
661
512
 	switch (map->bo_kmap_type) {
..
..
@@ -673,167 +524,116 @@
673
524
 	default:
674
525
 		BUG();
675
526
 	}
676
-	(void) ttm_mem_io_lock(man, false);
677
527
 	ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
678
-	ttm_mem_io_unlock(man);
679
528
 	map->virtual = NULL;
680
529
 	map->page = NULL;
681
530
 }
682
531
 EXPORT_SYMBOL(ttm_bo_kunmap);
683
532
 
533
+static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,
534
+				 bool dst_use_tt)
535
+{
536
+	int ret;
537
+	ret = ttm_bo_wait(bo, false, false);
538
+	if (ret)
539
+		return ret;
540
+
541
+	if (!dst_use_tt)
542
+		ttm_bo_tt_destroy(bo);
543
+	ttm_bo_free_old_node(bo);
544
+	return 0;
545
+}
546
+
547
+static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,
548
+				struct dma_fence *fence,
549
+				bool dst_use_tt)
550
+{
551
+	struct ttm_buffer_object *ghost_obj;
552
+	int ret;
553
+
554
+	/**
555
+	 * This should help pipeline ordinary buffer moves.
556
+	 *
557
+	 * Hang old buffer memory on a new buffer object,
558
+	 * and leave it to be released when the GPU
559
+	 * operation has completed.
560
+	 */
561
+
562
+	dma_fence_put(bo->moving);
563
+	bo->moving = dma_fence_get(fence);
564
+
565
+	ret = ttm_buffer_object_transfer(bo, &ghost_obj);
566
+	if (ret)
567
+		return ret;
568
+
569
+	dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
570
+
571
+	/**
572
+	 * If we're not moving to fixed memory, the TTM object
573
+	 * needs to stay alive. Otherwhise hang it on the ghost
574
+	 * bo to be unbound and destroyed.
575
+	 */
576
+
577
+	if (dst_use_tt)
578
+		ghost_obj->ttm = NULL;
579
+	else
580
+		bo->ttm = NULL;
581
+
582
+	dma_resv_unlock(&ghost_obj->base._resv);
583
+	ttm_bo_put(ghost_obj);
584
+	return 0;
585
+}
586
+
587
+static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,
588
+				       struct dma_fence *fence)
589
+{
590
+	struct ttm_bo_device *bdev = bo->bdev;
591
+	struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);
592
+
593
+	/**
594
+	 * BO doesn't have a TTM we need to bind/unbind. Just remember
595
+	 * this eviction and free up the allocation
596
+	 */
597
+	spin_lock(&from->move_lock);
598
+	if (!from->move || dma_fence_is_later(fence, from->move)) {
599
+		dma_fence_put(from->move);
600
+		from->move = dma_fence_get(fence);
601
+	}
602
+	spin_unlock(&from->move_lock);
603
+
604
+	ttm_bo_free_old_node(bo);
605
+
606
+	dma_fence_put(bo->moving);
607
+	bo->moving = dma_fence_get(fence);
608
+}
609
+
684
610
 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
685
611
 			      struct dma_fence *fence,
686
612
 			      bool evict,
687
-			      struct ttm_mem_reg *new_mem)
613
+			      bool pipeline,
614
+			      struct ttm_resource *new_mem)
688
615
 {
689
616
 	struct ttm_bo_device *bdev = bo->bdev;
690
-	struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
691
-	struct ttm_mem_reg *old_mem = &bo->mem;
692
-	int ret;
693
-	struct ttm_buffer_object *ghost_obj;
617
+	struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);
618
+	struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
619
+	int ret = 0;
694
620
 
695
-	reservation_object_add_excl_fence(bo->resv, fence);
696
-	if (evict) {
697
-		ret = ttm_bo_wait(bo, false, false);
698
-		if (ret)
699
-			return ret;
621
+	dma_resv_add_excl_fence(bo->base.resv, fence);
622
+	if (!evict)
623
+		ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt);
624
+	else if (!from->use_tt && pipeline)
625
+		ttm_bo_move_pipeline_evict(bo, fence);
626
+	else
627
+		ret = ttm_bo_wait_free_node(bo, man->use_tt);
700
628
 
701
-		if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
702
-			ttm_tt_destroy(bo->ttm);
703
-			bo->ttm = NULL;
704
-		}
705
-		ttm_bo_free_old_node(bo);
706
-	} else {
707
-		/**
708
-		 * This should help pipeline ordinary buffer moves.
709
-		 *
710
-		 * Hang old buffer memory on a new buffer object,
711
-		 * and leave it to be released when the GPU
712
-		 * operation has completed.
713
-		 */
629
+	if (ret)
630
+		return ret;
714
631
 
715
-		dma_fence_put(bo->moving);
716
-		bo->moving = dma_fence_get(fence);
717
-
718
-		ret = ttm_buffer_object_transfer(bo, &ghost_obj);
719
-		if (ret)
720
-			return ret;
721
-
722
-		reservation_object_add_excl_fence(ghost_obj->resv, fence);
723
-
724
-		/**
725
-		 * If we're not moving to fixed memory, the TTM object
726
-		 * needs to stay alive. Otherwhise hang it on the ghost
727
-		 * bo to be unbound and destroyed.
728
-		 */
729
-
730
-		if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
731
-			ghost_obj->ttm = NULL;
732
-		else
733
-			bo->ttm = NULL;
734
-
735
-		ttm_bo_unreserve(ghost_obj);
736
-		ttm_bo_put(ghost_obj);
737
-	}
738
-
739
-	*old_mem = *new_mem;
740
-	new_mem->mm_node = NULL;
632
+	ttm_bo_assign_mem(bo, new_mem);
741
633
 
742
634
 	return 0;
743
635
 }
744
636
 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
745
-
746
-int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
747
-			 struct dma_fence *fence, bool evict,
748
-			 struct ttm_mem_reg *new_mem)
749
-{
750
-	struct ttm_bo_device *bdev = bo->bdev;
751
-	struct ttm_mem_reg *old_mem = &bo->mem;
752
-
753
-	struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
754
-	struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
755
-
756
-	int ret;
757
-
758
-	reservation_object_add_excl_fence(bo->resv, fence);
759
-
760
-	if (!evict) {
761
-		struct ttm_buffer_object *ghost_obj;
762
-
763
-		/**
764
-		 * This should help pipeline ordinary buffer moves.
765
-		 *
766
-		 * Hang old buffer memory on a new buffer object,
767
-		 * and leave it to be released when the GPU
768
-		 * operation has completed.
769
-		 */
770
-
771
-		dma_fence_put(bo->moving);
772
-		bo->moving = dma_fence_get(fence);
773
-
774
-		ret = ttm_buffer_object_transfer(bo, &ghost_obj);
775
-		if (ret)
776
-			return ret;
777
-
778
-		reservation_object_add_excl_fence(ghost_obj->resv, fence);
779
-
780
-		/**
781
-		 * If we're not moving to fixed memory, the TTM object
782
-		 * needs to stay alive. Otherwhise hang it on the ghost
783
-		 * bo to be unbound and destroyed.
784
-		 */
785
-
786
-		if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
787
-			ghost_obj->ttm = NULL;
788
-		else
789
-			bo->ttm = NULL;
790
-
791
-		ttm_bo_unreserve(ghost_obj);
792
-		ttm_bo_put(ghost_obj);
793
-
794
-	} else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
795
-
796
-		/**
797
-		 * BO doesn't have a TTM we need to bind/unbind. Just remember
798
-		 * this eviction and free up the allocation
799
-		 */
800
-
801
-		spin_lock(&from->move_lock);
802
-		if (!from->move || dma_fence_is_later(fence, from->move)) {
803
-			dma_fence_put(from->move);
804
-			from->move = dma_fence_get(fence);
805
-		}
806
-		spin_unlock(&from->move_lock);
807
-
808
-		ttm_bo_free_old_node(bo);
809
-
810
-		dma_fence_put(bo->moving);
811
-		bo->moving = dma_fence_get(fence);
812
-
813
-	} else {
814
-		/**
815
-		 * Last resort, wait for the move to be completed.
816
-		 *
817
-		 * Should never happen in pratice.
818
-		 */
819
-
820
-		ret = ttm_bo_wait(bo, false, false);
821
-		if (ret)
822
-			return ret;
823
-
824
-		if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
825
-			ttm_tt_destroy(bo->ttm);
826
-			bo->ttm = NULL;
827
-		}
828
-		ttm_bo_free_old_node(bo);
829
-	}
830
-
831
-	*old_mem = *new_mem;
832
-	new_mem->mm_node = NULL;
833
-
834
-	return 0;
835
-}
836
-EXPORT_SYMBOL(ttm_bo_pipeline_move);
837
637
 
838
638
 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
839
639
 {
..
..
@@ -844,7 +644,7 @@
844
644
 	if (ret)
845
645
 		return ret;
846
646
 
847
-	ret = reservation_object_copy_fences(ghost->resv, bo->resv);
647
+	ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);
848
648
 	/* Last resort, wait for the BO to be idle when we are OOM */
849
649
 	if (ret)
850
650
 		ttm_bo_wait(bo, false, false);
..
..
@@ -853,7 +653,7 @@
853
653
 	bo->mem.mem_type = TTM_PL_SYSTEM;
854
654
 	bo->ttm = NULL;
855
655
 
856
-	ttm_bo_unreserve(ghost);
656
+	dma_resv_unlock(&ghost->base._resv);
857
657
 	ttm_bo_put(ghost);
858
658
 
859
659
 	return 0;