~hc/RK356X_SDK_RELEASE.git

..	..	@@ -29,28 +29,38 @@
29	29	* Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30	30	* Dave Airlie
31	31	*/
32		-#include <drm/ttm/ttm_bo_api.h>
33		-#include <drm/ttm/ttm_bo_driver.h>
34		-#include <drm/ttm/ttm_placement.h>
35		-#include <drm/ttm/ttm_module.h>
36		-#include <drm/ttm/ttm_page_alloc.h>
37		-#include <drm/drmP.h>
38		-#include <drm/radeon_drm.h>
	32	+
	33	+#include <linux/dma-mapping.h>
	34	+#include <linux/pagemap.h>
	35	+#include <linux/pci.h>
39	36	#include <linux/seq_file.h>
40	37	#include <linux/slab.h>
41		-#include <linux/swiotlb.h>
42	38	#include <linux/swap.h>
43		-#include <linux/pagemap.h>
44		-#include <linux/debugfs.h>
	39	+#include <linux/swiotlb.h>
	40	+
	41	+#include <drm/drm_agpsupport.h>
	42	+#include <drm/drm_debugfs.h>
	43	+#include <drm/drm_device.h>
	44	+#include <drm/drm_file.h>
	45	+#include <drm/drm_prime.h>
	46	+#include <drm/radeon_drm.h>
	47	+#include <drm/ttm/ttm_bo_api.h>
	48	+#include <drm/ttm/ttm_bo_driver.h>
	49	+#include <drm/ttm/ttm_module.h>
	50	+#include <drm/ttm/ttm_page_alloc.h>
	51	+#include <drm/ttm/ttm_placement.h>
	52	+
45	53	#include "radeon_reg.h"
46	54	#include "radeon.h"
47		-
48		-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
49	55
50	56	static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
51	57	static void radeon_ttm_debugfs_fini(struct radeon_device *rdev);
52	58
53		-static struct radeon_device radeon_get_rdev(struct ttm_bo_device bdev)
	59	+static int radeon_ttm_tt_bind(struct ttm_bo_device *bdev,
	60	+ struct ttm_tt *ttm,
	61	+ struct ttm_resource *bo_mem);
	62	+
	63	+struct radeon_device radeon_get_rdev(struct ttm_bo_device bdev)
54	64	{
55	65	struct radeon_mman *mman;
56	66	struct radeon_device *rdev;
..	..	@@ -60,119 +70,16 @@
60	70	return rdev;
61	71	}
62	72
63		-
64		-/*
65		- * Global memory.
66		- */
67		-static int radeon_ttm_mem_global_init(struct drm_global_reference *ref)
	73	+static int radeon_ttm_init_vram(struct radeon_device *rdev)
68	74	{
69		- return ttm_mem_global_init(ref->object);
	75	+ return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_VRAM,
	76	+ false, rdev->mc.real_vram_size >> PAGE_SHIFT);
70	77	}
71	78
72		-static void radeon_ttm_mem_global_release(struct drm_global_reference *ref)
	79	+static int radeon_ttm_init_gtt(struct radeon_device *rdev)
73	80	{
74		- ttm_mem_global_release(ref->object);
75		-}
76		-
77		-static int radeon_ttm_global_init(struct radeon_device *rdev)
78		-{
79		- struct drm_global_reference *global_ref;
80		- int r;
81		-
82		- rdev->mman.mem_global_referenced = false;
83		- global_ref = &rdev->mman.mem_global_ref;
84		- global_ref->global_type = DRM_GLOBAL_TTM_MEM;
85		- global_ref->size = sizeof(struct ttm_mem_global);
86		- global_ref->init = &radeon_ttm_mem_global_init;
87		- global_ref->release = &radeon_ttm_mem_global_release;
88		- r = drm_global_item_ref(global_ref);
89		- if (r != 0) {
90		- DRM_ERROR("Failed setting up TTM memory accounting "
91		- "subsystem.\n");
92		- return r;
93		- }
94		-
95		- rdev->mman.bo_global_ref.mem_glob =
96		- rdev->mman.mem_global_ref.object;
97		- global_ref = &rdev->mman.bo_global_ref.ref;
98		- global_ref->global_type = DRM_GLOBAL_TTM_BO;
99		- global_ref->size = sizeof(struct ttm_bo_global);
100		- global_ref->init = &ttm_bo_global_init;
101		- global_ref->release = &ttm_bo_global_release;
102		- r = drm_global_item_ref(global_ref);
103		- if (r != 0) {
104		- DRM_ERROR("Failed setting up TTM BO subsystem.\n");
105		- drm_global_item_unref(&rdev->mman.mem_global_ref);
106		- return r;
107		- }
108		-
109		- rdev->mman.mem_global_referenced = true;
110		- return 0;
111		-}
112		-
113		-static void radeon_ttm_global_fini(struct radeon_device *rdev)
114		-{
115		- if (rdev->mman.mem_global_referenced) {
116		- drm_global_item_unref(&rdev->mman.bo_global_ref.ref);
117		- drm_global_item_unref(&rdev->mman.mem_global_ref);
118		- rdev->mman.mem_global_referenced = false;
119		- }
120		-}
121		-
122		-static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
123		-{
124		- return 0;
125		-}
126		-
127		-static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
128		- struct ttm_mem_type_manager *man)
129		-{
130		- struct radeon_device *rdev;
131		-
132		- rdev = radeon_get_rdev(bdev);
133		-
134		- switch (type) {
135		- case TTM_PL_SYSTEM:
136		- /* System memory */
137		- man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
138		- man->available_caching = TTM_PL_MASK_CACHING;
139		- man->default_caching = TTM_PL_FLAG_CACHED;
140		- break;
141		- case TTM_PL_TT:
142		- man->func = &ttm_bo_manager_func;
143		- man->gpu_offset = rdev->mc.gtt_start;
144		- man->available_caching = TTM_PL_MASK_CACHING;
145		- man->default_caching = TTM_PL_FLAG_CACHED;
146		- man->flags = TTM_MEMTYPE_FLAG_MAPPABLE \| TTM_MEMTYPE_FLAG_CMA;
147		-#if IS_ENABLED(CONFIG_AGP)
148		- if (rdev->flags & RADEON_IS_AGP) {
149		- if (!rdev->ddev->agp) {
150		- DRM_ERROR("AGP is not enabled for memory type %u\n",
151		- (unsigned)type);
152		- return -EINVAL;
153		- }
154		- if (!rdev->ddev->agp->cant_use_aperture)
155		- man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
156		- man->available_caching = TTM_PL_FLAG_UNCACHED \|
157		- TTM_PL_FLAG_WC;
158		- man->default_caching = TTM_PL_FLAG_WC;
159		- }
160		-#endif
161		- break;
162		- case TTM_PL_VRAM:
163		- /* "On-card" video ram */
164		- man->func = &ttm_bo_manager_func;
165		- man->gpu_offset = rdev->mc.vram_start;
166		- man->flags = TTM_MEMTYPE_FLAG_FIXED \|
167		- TTM_MEMTYPE_FLAG_MAPPABLE;
168		- man->available_caching = TTM_PL_FLAG_UNCACHED \| TTM_PL_FLAG_WC;
169		- man->default_caching = TTM_PL_FLAG_WC;
170		- break;
171		- default:
172		- DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
173		- return -EINVAL;
174		- }
175		- return 0;
	81	+ return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_TT,
	82	+ true, rdev->mc.gtt_size >> PAGE_SHIFT);
176	83	}
177	84
178	85	static void radeon_evict_flags(struct ttm_buffer_object *bo,
..	..	@@ -181,7 +88,8 @@
181	88	static const struct ttm_place placements = {
182	89	.fpfn = 0,
183	90	.lpfn = 0,
184		- .flags = TTM_PL_MASK_CACHING \| TTM_PL_FLAG_SYSTEM
	91	+ .mem_type = TTM_PL_SYSTEM,
	92	+ .flags = TTM_PL_MASK_CACHING
185	93	};
186	94
187	95	struct radeon_bo *rbo;
..	..	@@ -212,7 +120,7 @@
212	120	RADEON_GEM_DOMAIN_GTT);
213	121	rbo->placement.num_busy_placement = 0;
214	122	for (i = 0; i < rbo->placement.num_placement; i++) {
215		- if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) {
	123	+ if (rbo->placements[i].mem_type == TTM_PL_VRAM) {
216	124	if (rbo->placements[i].fpfn < fpfn)
217	125	rbo->placements[i].fpfn = fpfn;
218	126	} else {
..	..	@@ -234,27 +142,18 @@
234	142	static int radeon_verify_access(struct ttm_buffer_object bo, struct file filp)
235	143	{
236	144	struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo);
	145	+ struct radeon_device *rdev = radeon_get_rdev(bo->bdev);
237	146
238		- if (radeon_ttm_tt_has_userptr(bo->ttm))
	147	+ if (radeon_ttm_tt_has_userptr(rdev, bo->ttm))
239	148	return -EPERM;
240		- return drm_vma_node_verify_access(&rbo->gem_base.vma_node,
	149	+ return drm_vma_node_verify_access(&rbo->tbo.base.vma_node,
241	150	filp->private_data);
242		-}
243		-
244		-static void radeon_move_null(struct ttm_buffer_object *bo,
245		- struct ttm_mem_reg *new_mem)
246		-{
247		- struct ttm_mem_reg *old_mem = &bo->mem;
248		-
249		- BUG_ON(old_mem->mm_node != NULL);
250		- old_mem = new_mem;
251		- new_mem->mm_node = NULL;
252	151	}
253	152
254	153	static int radeon_move_blit(struct ttm_buffer_object *bo,
255	154	bool evict, bool no_wait_gpu,
256		- struct ttm_mem_reg *new_mem,
257		- struct ttm_mem_reg *old_mem)
	155	+ struct ttm_resource *new_mem,
	156	+ struct ttm_resource *old_mem)
258	157	{
259	158	struct radeon_device *rdev;
260	159	uint64_t old_start, new_start;
..	..	@@ -297,11 +196,11 @@
297	196	BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);
298	197
299	198	num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
300		- fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->resv);
	199	+ fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv);
301	200	if (IS_ERR(fence))
302	201	return PTR_ERR(fence);
303	202
304		- r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem);
	203	+ r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, new_mem);
305	204	radeon_fence_unref(&fence);
306	205	return r;
307	206	}
..	..	@@ -309,17 +208,15 @@
309	208	static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
310	209	bool evict, bool interruptible,
311	210	bool no_wait_gpu,
312		- struct ttm_mem_reg *new_mem)
	211	+ struct ttm_resource *new_mem)
313	212	{
314	213	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
315		- struct radeon_device *rdev;
316		- struct ttm_mem_reg *old_mem = &bo->mem;
317		- struct ttm_mem_reg tmp_mem;
	214	+ struct ttm_resource *old_mem = &bo->mem;
	215	+ struct ttm_resource tmp_mem;
318	216	struct ttm_place placements;
319	217	struct ttm_placement placement;
320	218	int r;
321	219
322		- rdev = radeon_get_rdev(bo->bdev);
323	220	tmp_mem = *new_mem;
324	221	tmp_mem.mm_node = NULL;
325	222	placement.num_placement = 1;
..	..	@@ -328,7 +225,8 @@
328	225	placement.busy_placement = &placements;
329	226	placements.fpfn = 0;
330	227	placements.lpfn = 0;
331		- placements.flags = TTM_PL_MASK_CACHING \| TTM_PL_FLAG_TT;
	228	+ placements.mem_type = TTM_PL_TT;
	229	+ placements.flags = TTM_PL_MASK_CACHING;
332	230	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx);
333	231	if (unlikely(r)) {
334	232	return r;
..	..	@@ -339,7 +237,12 @@
339	237	goto out_cleanup;
340	238	}
341	239
342		- r = ttm_tt_bind(bo->ttm, &tmp_mem, &ctx);
	240	+ r = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);
	241	+ if (unlikely(r)) {
	242	+ goto out_cleanup;
	243	+ }
	244	+
	245	+ r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, &tmp_mem);
343	246	if (unlikely(r)) {
344	247	goto out_cleanup;
345	248	}
..	..	@@ -349,24 +252,22 @@
349	252	}
350	253	r = ttm_bo_move_ttm(bo, &ctx, new_mem);
351	254	out_cleanup:
352		- ttm_bo_mem_put(bo, &tmp_mem);
	255	+ ttm_resource_free(bo, &tmp_mem);
353	256	return r;
354	257	}
355	258
356	259	static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
357	260	bool evict, bool interruptible,
358	261	bool no_wait_gpu,
359		- struct ttm_mem_reg *new_mem)
	262	+ struct ttm_resource *new_mem)
360	263	{
361	264	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
362		- struct radeon_device *rdev;
363		- struct ttm_mem_reg *old_mem = &bo->mem;
364		- struct ttm_mem_reg tmp_mem;
	265	+ struct ttm_resource *old_mem = &bo->mem;
	266	+ struct ttm_resource tmp_mem;
365	267	struct ttm_placement placement;
366	268	struct ttm_place placements;
367	269	int r;
368	270
369		- rdev = radeon_get_rdev(bo->bdev);
370	271	tmp_mem = *new_mem;
371	272	tmp_mem.mm_node = NULL;
372	273	placement.num_placement = 1;
..	..	@@ -375,7 +276,8 @@
375	276	placement.busy_placement = &placements;
376	277	placements.fpfn = 0;
377	278	placements.lpfn = 0;
378		- placements.flags = TTM_PL_MASK_CACHING \| TTM_PL_FLAG_TT;
	279	+ placements.mem_type = TTM_PL_TT;
	280	+ placements.flags = TTM_PL_MASK_CACHING;
379	281	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx);
380	282	if (unlikely(r)) {
381	283	return r;
..	..	@@ -389,17 +291,17 @@
389	291	goto out_cleanup;
390	292	}
391	293	out_cleanup:
392		- ttm_bo_mem_put(bo, &tmp_mem);
	294	+ ttm_resource_free(bo, &tmp_mem);
393	295	return r;
394	296	}
395	297
396	298	static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict,
397	299	struct ttm_operation_ctx *ctx,
398		- struct ttm_mem_reg *new_mem)
	300	+ struct ttm_resource *new_mem)
399	301	{
400	302	struct radeon_device *rdev;
401	303	struct radeon_bo *rbo;
402		- struct ttm_mem_reg *old_mem = &bo->mem;
	304	+ struct ttm_resource *old_mem = &bo->mem;
403	305	int r;
404	306
405	307	r = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
..	..	@@ -413,7 +315,7 @@
413	315
414	316	rdev = radeon_get_rdev(bo->bdev);
415	317	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
416		- radeon_move_null(bo, new_mem);
	318	+ ttm_bo_move_null(bo, new_mem);
417	319	return 0;
418	320	}
419	321	if ((old_mem->mem_type == TTM_PL_TT &&
..	..	@@ -421,7 +323,7 @@
421	323	(old_mem->mem_type == TTM_PL_SYSTEM &&
422	324	new_mem->mem_type == TTM_PL_TT)) {
423	325	/* bind is enough */
424		- radeon_move_null(bo, new_mem);
	326	+ ttm_bo_move_null(bo, new_mem);
425	327	return 0;
426	328	}
427	329	if (!rdev->ring[radeon_copy_ring_index(rdev)].ready \|\|
..	..	@@ -456,18 +358,11 @@
456	358	return 0;
457	359	}
458	360
459		-static int radeon_ttm_io_mem_reserve(struct ttm_bo_device bdev, struct ttm_mem_reg mem)
	361	+static int radeon_ttm_io_mem_reserve(struct ttm_bo_device bdev, struct ttm_resource mem)
460	362	{
461		- struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
462	363	struct radeon_device *rdev = radeon_get_rdev(bdev);
	364	+ size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
463	365
464		- mem->bus.addr = NULL;
465		- mem->bus.offset = 0;
466		- mem->bus.size = mem->num_pages << PAGE_SHIFT;
467		- mem->bus.base = 0;
468		- mem->bus.is_iomem = false;
469		- if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
470		- return -EINVAL;
471	366	switch (mem->mem_type) {
472	367	case TTM_PL_SYSTEM:
473	368	/* system memory */
..	..	@@ -476,8 +371,8 @@
476	371	#if IS_ENABLED(CONFIG_AGP)
477	372	if (rdev->flags & RADEON_IS_AGP) {
478	373	/* RADEON_IS_AGP is set only if AGP is active */
479		- mem->bus.offset = mem->start << PAGE_SHIFT;
480		- mem->bus.base = rdev->mc.agp_base;
	374	+ mem->bus.offset = (mem->start << PAGE_SHIFT) +
	375	+ rdev->mc.agp_base;
481	376	mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture;
482	377	}
483	378	#endif
..	..	@@ -485,9 +380,9 @@
485	380	case TTM_PL_VRAM:
486	381	mem->bus.offset = mem->start << PAGE_SHIFT;
487	382	/* check if it's visible */
488		- if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size)
	383	+ if ((mem->bus.offset + bus_size) > rdev->mc.visible_vram_size)
489	384	return -EINVAL;
490		- mem->bus.base = rdev->mc.aper_base;
	385	+ mem->bus.offset += rdev->mc.aper_base;
491	386	mem->bus.is_iomem = true;
492	387	#ifdef __alpha__
493	388	/*
..	..	@@ -496,12 +391,10 @@
496	391	*/
497	392	if (mem->placement & TTM_PL_FLAG_WC)
498	393	mem->bus.addr =
499		- ioremap_wc(mem->bus.base + mem->bus.offset,
500		- mem->bus.size);
	394	+ ioremap_wc(mem->bus.offset, bus_size);
501	395	else
502	396	mem->bus.addr =
503		- ioremap_nocache(mem->bus.base + mem->bus.offset,
504		- mem->bus.size);
	397	+ ioremap(mem->bus.offset, bus_size);
505	398	if (!mem->bus.addr)
506	399	return -ENOMEM;
507	400
..	..	@@ -511,7 +404,7 @@
511	404	* It then can be used to build PTEs for VRAM
512	405	* access, as done in ttm_bo_vm_fault().
513	406	*/
514		- mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
	407	+ mem->bus.offset = (mem->bus.offset & 0x0ffffffffUL) +
515	408	rdev->ddev->hose->dense_mem_base;
516	409	#endif
517	410	break;
..	..	@@ -521,29 +414,25 @@
521	414	return 0;
522	415	}
523	416
524		-static void radeon_ttm_io_mem_free(struct ttm_bo_device bdev, struct ttm_mem_reg mem)
525		-{
526		-}
527		-
528	417	/*
529	418	* TTM backend functions.
530	419	*/
531	420	struct radeon_ttm_tt {
532	421	struct ttm_dma_tt ttm;
533		- struct radeon_device *rdev;
534	422	u64 offset;
535	423
536	424	uint64_t userptr;
537	425	struct mm_struct *usermm;
538	426	uint32_t userflags;
	427	+ bool bound;
539	428	};
540	429
541	430	/* prepare the sg table with the user pages */
542		-static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm)
	431	+static int radeon_ttm_tt_pin_userptr(struct ttm_bo_device bdev, struct ttm_tt ttm)
543	432	{
544		- struct radeon_device *rdev = radeon_get_rdev(ttm->bdev);
	433	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
545	434	struct radeon_ttm_tt gtt = (void )ttm;
546		- unsigned pinned = 0, nents;
	435	+ unsigned pinned = 0;
547	436	int r;
548	437
549	438	int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY);
..	..	@@ -583,9 +472,8 @@
583	472	if (r)
584	473	goto release_sg;
585	474
586		- r = -ENOMEM;
587		- nents = dma_map_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
588		- if (nents != ttm->sg->nents)
	475	+ r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0);
	476	+ if (r)
589	477	goto release_sg;
590	478
591	479	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
..	..	@@ -601,9 +489,9 @@
601	489	return r;
602	490	}
603	491
604		-static void radeon_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
	492	+static void radeon_ttm_tt_unpin_userptr(struct ttm_bo_device bdev, struct ttm_tt ttm)
605	493	{
606		- struct radeon_device *rdev = radeon_get_rdev(ttm->bdev);
	494	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
607	495	struct radeon_ttm_tt gtt = (void )ttm;
608	496	struct sg_page_iter sg_iter;
609	497
..	..	@@ -616,9 +504,9 @@
616	504	return;
617	505
618	506	/* free the sg table and pages again */
619		- dma_unmap_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
	507	+ dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0);
620	508
621		- for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) {
	509	+ for_each_sgtable_page(ttm->sg, &sg_iter, 0) {
622	510	struct page *page = sg_page_iter_page(&sg_iter);
623	511	if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY))
624	512	set_page_dirty(page);
..	..	@@ -630,16 +518,28 @@
630	518	sg_free_table(ttm->sg);
631	519	}
632	520
633		-static int radeon_ttm_backend_bind(struct ttm_tt *ttm,
634		- struct ttm_mem_reg *bo_mem)
	521	+static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm)
635	522	{
636	523	struct radeon_ttm_tt gtt = (void)ttm;
	524	+
	525	+ return (gtt->bound);
	526	+}
	527	+
	528	+static int radeon_ttm_backend_bind(struct ttm_bo_device *bdev,
	529	+ struct ttm_tt *ttm,
	530	+ struct ttm_resource *bo_mem)
	531	+{
	532	+ struct radeon_ttm_tt gtt = (void)ttm;
	533	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
637	534	uint32_t flags = RADEON_GART_PAGE_VALID \| RADEON_GART_PAGE_READ \|
638	535	RADEON_GART_PAGE_WRITE;
639	536	int r;
640	537
	538	+ if (gtt->bound)
	539	+ return 0;
	540	+
641	541	if (gtt->userptr) {
642		- radeon_ttm_tt_pin_userptr(ttm);
	542	+ radeon_ttm_tt_pin_userptr(bdev, ttm);
643	543	flags &= ~RADEON_GART_PAGE_WRITE;
644	544	}
645	545
..	..	@@ -650,41 +550,43 @@
650	550	}
651	551	if (ttm->caching_state == tt_cached)
652	552	flags \|= RADEON_GART_PAGE_SNOOP;
653		- r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages,
	553	+ r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages,
654	554	ttm->pages, gtt->ttm.dma_address, flags);
655	555	if (r) {
656	556	DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
657	557	ttm->num_pages, (unsigned)gtt->offset);
658	558	return r;
659	559	}
	560	+ gtt->bound = true;
660	561	return 0;
661	562	}
662	563
663		-static int radeon_ttm_backend_unbind(struct ttm_tt *ttm)
	564	+static void radeon_ttm_backend_unbind(struct ttm_bo_device bdev, struct ttm_tt ttm)
664	565	{
665	566	struct radeon_ttm_tt gtt = (void )ttm;
666		-
667		- radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages);
	567	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
668	568
669	569	if (gtt->userptr)
670		- radeon_ttm_tt_unpin_userptr(ttm);
	570	+ radeon_ttm_tt_unpin_userptr(bdev, ttm);
671	571
672		- return 0;
	572	+ if (!gtt->bound)
	573	+ return;
	574	+
	575	+ radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages);
	576	+
	577	+ gtt->bound = false;
673	578	}
674	579
675		-static void radeon_ttm_backend_destroy(struct ttm_tt *ttm)
	580	+static void radeon_ttm_backend_destroy(struct ttm_bo_device bdev, struct ttm_tt ttm)
676	581	{
677	582	struct radeon_ttm_tt gtt = (void )ttm;
	583	+
	584	+ radeon_ttm_backend_unbind(bdev, ttm);
	585	+ ttm_tt_destroy_common(bdev, ttm);
678	586
679	587	ttm_dma_tt_fini(&gtt->ttm);
680	588	kfree(gtt);
681	589	}
682		-
683		-static struct ttm_backend_func radeon_backend_func = {
684		- .bind = &radeon_ttm_backend_bind,
685		- .unbind = &radeon_ttm_backend_unbind,
686		- .destroy = &radeon_ttm_backend_destroy,
687		-};
688	590
689	591	static struct ttm_tt radeon_ttm_tt_create(struct ttm_buffer_object bo,
690	592	uint32_t page_flags)
..	..	@@ -704,8 +606,6 @@
704	606	if (gtt == NULL) {
705	607	return NULL;
706	608	}
707		- gtt->ttm.ttm.func = &radeon_backend_func;
708		- gtt->rdev = rdev;
709	609	if (ttm_dma_tt_init(&gtt->ttm, bo, page_flags)) {
710	610	kfree(gtt);
711	611	return NULL;
..	..	@@ -713,18 +613,25 @@
713	613	return &gtt->ttm.ttm;
714	614	}
715	615
716		-static struct radeon_ttm_tt radeon_ttm_tt_to_gtt(struct ttm_tt ttm)
	616	+static struct radeon_ttm_tt radeon_ttm_tt_to_gtt(struct radeon_device rdev,
	617	+ struct ttm_tt *ttm)
717	618	{
718		- if (!ttm \|\| ttm->func != &radeon_backend_func)
	619	+#if IS_ENABLED(CONFIG_AGP)
	620	+ if (rdev->flags & RADEON_IS_AGP)
719	621	return NULL;
720		- return (struct radeon_ttm_tt *)ttm;
	622	+#endif
	623	+
	624	+ if (!ttm)
	625	+ return NULL;
	626	+ return container_of(ttm, struct radeon_ttm_tt, ttm.ttm);
721	627	}
722	628
723		-static int radeon_ttm_tt_populate(struct ttm_tt *ttm,
724		- struct ttm_operation_ctx *ctx)
	629	+static int radeon_ttm_tt_populate(struct ttm_bo_device *bdev,
	630	+ struct ttm_tt *ttm,
	631	+ struct ttm_operation_ctx *ctx)
725	632	{
726		- struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
727		- struct radeon_device *rdev;
	633	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	634	+ struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);
728	635	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
729	636
730	637	if (gtt && gtt->userptr) {
..	..	@@ -733,21 +640,20 @@
733	640	return -ENOMEM;
734	641
735	642	ttm->page_flags \|= TTM_PAGE_FLAG_SG;
736		- ttm->state = tt_unbound;
	643	+ ttm_tt_set_populated(ttm);
737	644	return 0;
738	645	}
739	646
740	647	if (slave && ttm->sg) {
741	648	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
742	649	gtt->ttm.dma_address, ttm->num_pages);
743		- ttm->state = tt_unbound;
	650	+ ttm_tt_set_populated(ttm);
744	651	return 0;
745	652	}
746	653
747		- rdev = radeon_get_rdev(ttm->bdev);
748	654	#if IS_ENABLED(CONFIG_AGP)
749	655	if (rdev->flags & RADEON_IS_AGP) {
750		- return ttm_agp_tt_populate(ttm, ctx);
	656	+ return ttm_pool_populate(ttm, ctx);
751	657	}
752	658	#endif
753	659
..	..	@@ -760,10 +666,10 @@
760	666	return ttm_populate_and_map_pages(rdev->dev, &gtt->ttm, ctx);
761	667	}
762	668
763		-static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
	669	+static void radeon_ttm_tt_unpopulate(struct ttm_bo_device bdev, struct ttm_tt ttm)
764	670	{
765		- struct radeon_device *rdev;
766		- struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
	671	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	672	+ struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);
767	673	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
768	674
769	675	if (gtt && gtt->userptr) {
..	..	@@ -775,10 +681,9 @@
775	681	if (slave)
776	682	return;
777	683
778		- rdev = radeon_get_rdev(ttm->bdev);
779	684	#if IS_ENABLED(CONFIG_AGP)
780	685	if (rdev->flags & RADEON_IS_AGP) {
781		- ttm_agp_tt_unpopulate(ttm);
	686	+ ttm_pool_unpopulate(ttm);
782	687	return;
783	688	}
784	689	#endif
..	..	@@ -793,10 +698,11 @@
793	698	ttm_unmap_and_unpopulate_pages(rdev->dev, &gtt->ttm);
794	699	}
795	700
796		-int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
	701	+int radeon_ttm_tt_set_userptr(struct radeon_device *rdev,
	702	+ struct ttm_tt *ttm, uint64_t addr,
797	703	uint32_t flags)
798	704	{
799		- struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
	705	+ struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);
800	706
801	707	if (gtt == NULL)
802	708	return -EINVAL;
..	..	@@ -807,9 +713,69 @@
807	713	return 0;
808	714	}
809	715
810		-bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm)
	716	+bool radeon_ttm_tt_is_bound(struct ttm_bo_device *bdev,
	717	+ struct ttm_tt *ttm)
811	718	{
812		- struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
	719	+#if IS_ENABLED(CONFIG_AGP)
	720	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	721	+ if (rdev->flags & RADEON_IS_AGP)
	722	+ return ttm_agp_is_bound(ttm);
	723	+#endif
	724	+ return radeon_ttm_backend_is_bound(ttm);
	725	+}
	726	+
	727	+static int radeon_ttm_tt_bind(struct ttm_bo_device *bdev,
	728	+ struct ttm_tt *ttm,
	729	+ struct ttm_resource *bo_mem)
	730	+{
	731	+#if IS_ENABLED(CONFIG_AGP)
	732	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	733	+#endif
	734	+
	735	+ if (!bo_mem)
	736	+ return -EINVAL;
	737	+#if IS_ENABLED(CONFIG_AGP)
	738	+ if (rdev->flags & RADEON_IS_AGP)
	739	+ return ttm_agp_bind(ttm, bo_mem);
	740	+#endif
	741	+
	742	+ return radeon_ttm_backend_bind(bdev, ttm, bo_mem);
	743	+}
	744	+
	745	+static void radeon_ttm_tt_unbind(struct ttm_bo_device *bdev,
	746	+ struct ttm_tt *ttm)
	747	+{
	748	+#if IS_ENABLED(CONFIG_AGP)
	749	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	750	+
	751	+ if (rdev->flags & RADEON_IS_AGP) {
	752	+ ttm_agp_unbind(ttm);
	753	+ return;
	754	+ }
	755	+#endif
	756	+ radeon_ttm_backend_unbind(bdev, ttm);
	757	+}
	758	+
	759	+static void radeon_ttm_tt_destroy(struct ttm_bo_device *bdev,
	760	+ struct ttm_tt *ttm)
	761	+{
	762	+#if IS_ENABLED(CONFIG_AGP)
	763	+ struct radeon_device *rdev = radeon_get_rdev(bdev);
	764	+
	765	+ if (rdev->flags & RADEON_IS_AGP) {
	766	+ ttm_agp_unbind(ttm);
	767	+ ttm_tt_destroy_common(bdev, ttm);
	768	+ ttm_agp_destroy(ttm);
	769	+ return;
	770	+ }
	771	+#endif
	772	+ radeon_ttm_backend_destroy(bdev, ttm);
	773	+}
	774	+
	775	+bool radeon_ttm_tt_has_userptr(struct radeon_device *rdev,
	776	+ struct ttm_tt *ttm)
	777	+{
	778	+ struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);
813	779
814	780	if (gtt == NULL)
815	781	return false;
..	..	@@ -817,9 +783,10 @@
817	783	return !!gtt->userptr;
818	784	}
819	785
820		-bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm)
	786	+bool radeon_ttm_tt_is_readonly(struct radeon_device *rdev,
	787	+ struct ttm_tt *ttm)
821	788	{
822		- struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
	789	+ struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);
823	790
824	791	if (gtt == NULL)
825	792	return false;
..	..	@@ -831,8 +798,9 @@
831	798	.ttm_tt_create = &radeon_ttm_tt_create,
832	799	.ttm_tt_populate = &radeon_ttm_tt_populate,
833	800	.ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate,
834		- .invalidate_caches = &radeon_invalidate_caches,
835		- .init_mem_type = &radeon_init_mem_type,
	801	+ .ttm_tt_bind = &radeon_ttm_tt_bind,
	802	+ .ttm_tt_unbind = &radeon_ttm_tt_unbind,
	803	+ .ttm_tt_destroy = &radeon_ttm_tt_destroy,
836	804	.eviction_valuable = ttm_bo_eviction_valuable,
837	805	.evict_flags = &radeon_evict_flags,
838	806	.move = &radeon_bo_move,
..	..	@@ -840,31 +808,25 @@
840	808	.move_notify = &radeon_bo_move_notify,
841	809	.fault_reserve_notify = &radeon_bo_fault_reserve_notify,
842	810	.io_mem_reserve = &radeon_ttm_io_mem_reserve,
843		- .io_mem_free = &radeon_ttm_io_mem_free,
844	811	};
845	812
846	813	int radeon_ttm_init(struct radeon_device *rdev)
847	814	{
848	815	int r;
849	816
850		- r = radeon_ttm_global_init(rdev);
851		- if (r) {
852		- return r;
853		- }
854	817	/* No others user of address space so set it to 0 */
855	818	r = ttm_bo_device_init(&rdev->mman.bdev,
856		- rdev->mman.bo_global_ref.ref.object,
857	819	&radeon_bo_driver,
858	820	rdev->ddev->anon_inode->i_mapping,
859		- DRM_FILE_PAGE_OFFSET,
860		- rdev->need_dma32);
	821	+ rdev->ddev->vma_offset_manager,
	822	+ dma_addressing_limited(&rdev->pdev->dev));
861	823	if (r) {
862	824	DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
863	825	return r;
864	826	}
865	827	rdev->mman.initialized = true;
866		- r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM,
867		- rdev->mc.real_vram_size >> PAGE_SHIFT);
	828	+
	829	+ r = radeon_ttm_init_vram(rdev);
868	830	if (r) {
869	831	DRM_ERROR("Failed initializing VRAM heap.\n");
870	832	return r;
..	..	@@ -889,8 +851,8 @@
889	851	}
890	852	DRM_INFO("radeon: %uM of VRAM memory ready\n",
891	853	(unsigned) (rdev->mc.real_vram_size / (1024 * 1024)));
892		- r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT,
893		- rdev->mc.gtt_size >> PAGE_SHIFT);
	854	+
	855	+ r = radeon_ttm_init_gtt(rdev);
894	856	if (r) {
895	857	DRM_ERROR("Failed initializing GTT heap.\n");
896	858	return r;
..	..	@@ -921,11 +883,10 @@
921	883	}
922	884	radeon_bo_unref(&rdev->stolen_vga_memory);
923	885	}
924		- ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
925		- ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
	886	+ ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_VRAM);
	887	+ ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_TT);
926	888	ttm_bo_device_release(&rdev->mman.bdev);
927	889	radeon_gart_fini(rdev);
928		- radeon_ttm_global_fini(rdev);
929	890	rdev->mman.initialized = false;
930	891	DRM_INFO("radeon: ttm finalized\n");
931	892	}
..	..	@@ -934,18 +895,15 @@
934	895	* isn't running */
935	896	void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size)
936	897	{
937		- struct ttm_mem_type_manager *man;
	898	+ struct ttm_resource_manager *man;
938	899
939	900	if (!rdev->mman.initialized)
940	901	return;
941	902
942		- man = &rdev->mman.bdev.man[TTM_PL_VRAM];
	903	+ man = ttm_manager_type(&rdev->mman.bdev, TTM_PL_VRAM);
943	904	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
944	905	man->size = size >> PAGE_SHIFT;
945	906	}
946		-
947		-static struct vm_operations_struct radeon_ttm_vm_ops;
948		-static const struct vm_operations_struct *ttm_vm_ops = NULL;
949	907
950	908	static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf)
951	909	{
..	..	@@ -954,40 +912,36 @@
954	912	vm_fault_t ret;
955	913
956	914	bo = (struct ttm_buffer_object *)vmf->vma->vm_private_data;
957		- if (bo == NULL) {
	915	+ if (bo == NULL)
958	916	return VM_FAULT_NOPAGE;
959		- }
	917	+
960	918	rdev = radeon_get_rdev(bo->bdev);
961	919	down_read(&rdev->pm.mclk_lock);
962		- ret = ttm_vm_ops->fault(vmf);
	920	+ ret = ttm_bo_vm_fault(vmf);
963	921	up_read(&rdev->pm.mclk_lock);
964	922	return ret;
965	923	}
966	924
	925	+static struct vm_operations_struct radeon_ttm_vm_ops = {
	926	+ .fault = radeon_ttm_fault,
	927	+ .open = ttm_bo_vm_open,
	928	+ .close = ttm_bo_vm_close,
	929	+ .access = ttm_bo_vm_access
	930	+};
	931	+
967	932	int radeon_mmap(struct file filp, struct vm_area_struct vma)
968	933	{
969		- struct drm_file *file_priv;
970		- struct radeon_device *rdev;
971	934	int r;
	935	+ struct drm_file *file_priv = filp->private_data;
	936	+ struct radeon_device *rdev = file_priv->minor->dev->dev_private;
972	937
973		- if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
	938	+ if (rdev == NULL)
974	939	return -EINVAL;
975		- }
976	940
977		- file_priv = filp->private_data;
978		- rdev = file_priv->minor->dev->dev_private;
979		- if (rdev == NULL) {
980		- return -EINVAL;
981		- }
982	941	r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev);
983		- if (unlikely(r != 0)) {
	942	+ if (unlikely(r != 0))
984	943	return r;
985		- }
986		- if (unlikely(ttm_vm_ops == NULL)) {
987		- ttm_vm_ops = vma->vm_ops;
988		- radeon_ttm_vm_ops = *ttm_vm_ops;
989		- radeon_ttm_vm_ops.fault = &radeon_ttm_fault;
990		- }
	944	+
991	945	vma->vm_ops = &radeon_ttm_vm_ops;
992	946	return 0;
993	947	}
..	..	@@ -1000,7 +954,7 @@
1000	954	unsigned ttm_pl = (int)node->info_ent->data;
1001	955	struct drm_device *dev = node->minor->dev;
1002	956	struct radeon_device *rdev = dev->dev_private;
1003		- struct ttm_mem_type_manager *man = &rdev->mman.bdev.man[ttm_pl];
	957	+ struct ttm_resource_manager *man = ttm_manager_type(&rdev->mman.bdev, ttm_pl);
1004	958	struct drm_printer p = drm_seq_file_printer(m);
1005	959
1006	960	man->func->debug(man, &p);
..	..	@@ -1134,19 +1088,14 @@
1134	1088	unsigned count;
1135	1089
1136	1090	struct drm_minor *minor = rdev->ddev->primary;
1137		- struct dentry ent, root = minor->debugfs_root;
	1091	+ struct dentry *root = minor->debugfs_root;
1138	1092
1139		- ent = debugfs_create_file("radeon_vram", S_IFREG \| S_IRUGO, root,
1140		- rdev, &radeon_ttm_vram_fops);
1141		- if (IS_ERR(ent))
1142		- return PTR_ERR(ent);
1143		- rdev->mman.vram = ent;
	1093	+ rdev->mman.vram = debugfs_create_file("radeon_vram", S_IFREG \| S_IRUGO,
	1094	+ root, rdev,
	1095	+ &radeon_ttm_vram_fops);
1144	1096
1145		- ent = debugfs_create_file("radeon_gtt", S_IFREG \| S_IRUGO, root,
1146		- rdev, &radeon_ttm_gtt_fops);
1147		- if (IS_ERR(ent))
1148		- return PTR_ERR(ent);
1149		- rdev->mman.gtt = ent;
	1097	+ rdev->mman.gtt = debugfs_create_file("radeon_gtt", S_IFREG \| S_IRUGO,
	1098	+ root, rdev, &radeon_ttm_gtt_fops);
1150	1099
1151	1100	count = ARRAY_SIZE(radeon_ttm_debugfs_list);
1152	1101