forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/drivers/xen/swiotlb-xen.c
....@@ -1,17 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *  Copyright 2010
34  *  by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
45  *
56  * This code provides a IOMMU for Xen PV guests with PCI passthrough.
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License v2.0 as published by
9
- * the Free Software Foundation
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
157  *
168  * PV guests under Xen are running in an non-contiguous memory architecture.
179  *
....@@ -30,13 +22,13 @@
3022  * and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are
3123  * allocated in descending order (high to low), meaning the guest might
3224  * never get any MFN's under the 4GB mark.
33
- *
3425  */
3526 
3627 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
3728 
38
-#include <linux/bootmem.h>
29
+#include <linux/memblock.h>
3930 #include <linux/dma-direct.h>
31
+#include <linux/dma-map-ops.h>
4032 #include <linux/export.h>
4133 #include <xen/swiotlb-xen.h>
4234 #include <xen/page.h>
....@@ -47,13 +39,12 @@
4739 #include <asm/xen/page-coherent.h>
4840 
4941 #include <trace/events/swiotlb.h>
42
+#define MAX_DMA_BITS 32
5043 /*
5144  * Used to do a quick range check in swiotlb_tbl_unmap_single and
5245  * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
5346  * API.
5447  */
55
-
56
-#define XEN_SWIOTLB_ERROR_CODE	(~(dma_addr_t)0x0)
5748 
5849 static char *xen_io_tlb_start, *xen_io_tlb_end;
5950 static unsigned long xen_io_tlb_nslabs;
....@@ -61,74 +52,60 @@
6152  * Quick lookup value of the bus address of the IOTLB.
6253  */
6354 
64
-static u64 start_dma_addr;
65
-
66
-/*
67
- * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t
68
- * can be 32bit when dma_addr_t is 64bit leading to a loss in
69
- * information if the shift is done before casting to 64bit.
70
- */
71
-static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
55
+static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr)
7256 {
7357 	unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr));
74
-	dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT;
58
+	phys_addr_t baddr = (phys_addr_t)bfn << XEN_PAGE_SHIFT;
7559 
76
-	dma |= paddr & ~XEN_PAGE_MASK;
77
-
78
-	return dma;
60
+	baddr |= paddr & ~XEN_PAGE_MASK;
61
+	return baddr;
7962 }
8063 
81
-static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr)
64
+static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr)
65
+{
66
+	return phys_to_dma(dev, xen_phys_to_bus(dev, paddr));
67
+}
68
+
69
+static inline phys_addr_t xen_bus_to_phys(struct device *dev,
70
+					  phys_addr_t baddr)
8271 {
8372 	unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr));
84
-	dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT;
85
-	phys_addr_t paddr = dma;
86
-
87
-	paddr |= baddr & ~XEN_PAGE_MASK;
73
+	phys_addr_t paddr = (xen_pfn << XEN_PAGE_SHIFT) |
74
+			    (baddr & ~XEN_PAGE_MASK);
8875 
8976 	return paddr;
9077 }
9178 
92
-static inline dma_addr_t xen_virt_to_bus(void *address)
79
+static inline phys_addr_t xen_dma_to_phys(struct device *dev,
80
+					  dma_addr_t dma_addr)
9381 {
94
-	return xen_phys_to_bus(virt_to_phys(address));
82
+	return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr));
9583 }
9684 
97
-static int check_pages_physically_contiguous(unsigned long xen_pfn,
98
-					     unsigned int offset,
99
-					     size_t length)
85
+static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address)
10086 {
101
-	unsigned long next_bfn;
102
-	int i;
103
-	int nr_pages;
104
-
105
-	next_bfn = pfn_to_bfn(xen_pfn);
106
-	nr_pages = (offset + length + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT;
107
-
108
-	for (i = 1; i < nr_pages; i++) {
109
-		if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
110
-			return 0;
111
-	}
112
-	return 1;
87
+	return xen_phys_to_dma(dev, virt_to_phys(address));
11388 }
11489 
11590 static inline int range_straddles_page_boundary(phys_addr_t p, size_t size)
11691 {
117
-	unsigned long xen_pfn = XEN_PFN_DOWN(p);
118
-	unsigned int offset = p & ~XEN_PAGE_MASK;
92
+	unsigned long next_bfn, xen_pfn = XEN_PFN_DOWN(p);
93
+	unsigned int i, nr_pages = XEN_PFN_UP(xen_offset_in_page(p) + size);
11994 
120
-	if (offset + size <= XEN_PAGE_SIZE)
121
-		return 0;
122
-	if (check_pages_physically_contiguous(xen_pfn, offset, size))
123
-		return 0;
124
-	return 1;
95
+	next_bfn = pfn_to_bfn(xen_pfn);
96
+
97
+	for (i = 1; i < nr_pages; i++)
98
+		if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
99
+			return 1;
100
+
101
+	return 0;
125102 }
126103 
127
-static int is_xen_swiotlb_buffer(dma_addr_t dma_addr)
104
+static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
128105 {
129
-	unsigned long bfn = XEN_PFN_DOWN(dma_addr);
106
+	unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
130107 	unsigned long xen_pfn = bfn_to_local_pfn(bfn);
131
-	phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn);
108
+	phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT;
132109 
133110 	/* If the address is outside our domain, it CAN
134111 	 * have the same virtual address as another address
....@@ -140,8 +117,6 @@
140117 	}
141118 	return 0;
142119 }
143
-
144
-static int max_dma_bits = 32;
145120 
146121 static int
147122 xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
....@@ -162,7 +137,7 @@
162137 				p + (i << IO_TLB_SHIFT),
163138 				get_order(slabs << IO_TLB_SHIFT),
164139 				dma_bits, &dma_handle);
165
-		} while (rc && dma_bits++ < max_dma_bits);
140
+		} while (rc && dma_bits++ < MAX_DMA_BITS);
166141 		if (rc)
167142 			return rc;
168143 
....@@ -213,12 +188,25 @@
213188 retry:
214189 	bytes = xen_set_nslabs(xen_io_tlb_nslabs);
215190 	order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
191
+
192
+	/*
193
+	 * IO TLB memory already allocated. Just use it.
194
+	 */
195
+	if (io_tlb_start != 0) {
196
+		xen_io_tlb_start = phys_to_virt(io_tlb_start);
197
+		goto end;
198
+	}
199
+
216200 	/*
217201 	 * Get IO TLB memory from any location.
218202 	 */
219
-	if (early)
220
-		xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
221
-	else {
203
+	if (early) {
204
+		xen_io_tlb_start = memblock_alloc(PAGE_ALIGN(bytes),
205
+						  PAGE_SIZE);
206
+		if (!xen_io_tlb_start)
207
+			panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
208
+			      __func__, PAGE_ALIGN(bytes), PAGE_SIZE);
209
+	} else {
222210 #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
223211 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
224212 		while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
....@@ -238,7 +226,6 @@
238226 		m_ret = XEN_SWIOTLB_ENOMEM;
239227 		goto error;
240228 	}
241
-	xen_io_tlb_end = xen_io_tlb_start + bytes;
242229 	/*
243230 	 * And replace that memory with pages under 4GB.
244231 	 */
....@@ -247,7 +234,8 @@
247234 			       xen_io_tlb_nslabs);
248235 	if (rc) {
249236 		if (early)
250
-			free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
237
+			memblock_free(__pa(xen_io_tlb_start),
238
+				      PAGE_ALIGN(bytes));
251239 		else {
252240 			free_pages((unsigned long)xen_io_tlb_start, order);
253241 			xen_io_tlb_start = NULL;
....@@ -255,7 +243,6 @@
255243 		m_ret = XEN_SWIOTLB_EFIXUP;
256244 		goto error;
257245 	}
258
-	start_dma_addr = xen_virt_to_bus(xen_io_tlb_start);
259246 	if (early) {
260247 		if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs,
261248 			 verbose))
....@@ -264,6 +251,8 @@
264251 	} else
265252 		rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
266253 
254
+end:
255
+	xen_io_tlb_end = xen_io_tlb_start + bytes;
267256 	if (!rc)
268257 		swiotlb_set_max_segment(PAGE_SIZE);
269258 
....@@ -319,12 +308,12 @@
319308 	if (hwdev && hwdev->coherent_dma_mask)
320309 		dma_mask = hwdev->coherent_dma_mask;
321310 
322
-	/* At this point dma_handle is the physical address, next we are
311
+	/* At this point dma_handle is the dma address, next we are
323312 	 * going to set it to the machine address.
324313 	 * Do not use virt_to_phys(ret) because on ARM it doesn't correspond
325314 	 * to *dma_handle. */
326
-	phys = *dma_handle;
327
-	dev_addr = xen_phys_to_bus(phys);
315
+	phys = dma_to_phys(hwdev, *dma_handle);
316
+	dev_addr = xen_phys_to_dma(hwdev, phys);
328317 	if (((dev_addr + size - 1 <= dma_mask)) &&
329318 	    !range_straddles_page_boundary(phys, size))
330319 		*dma_handle = dev_addr;
....@@ -334,6 +323,8 @@
334323 			xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs);
335324 			return NULL;
336325 		}
326
+		*dma_handle = phys_to_dma(hwdev, *dma_handle);
327
+		SetPageXenRemapped(virt_to_page(ret));
337328 	}
338329 	memset(ret, 0, size);
339330 	return ret;
....@@ -346,22 +337,30 @@
346337 	int order = get_order(size);
347338 	phys_addr_t phys;
348339 	u64 dma_mask = DMA_BIT_MASK(32);
340
+	struct page *page;
349341 
350342 	if (hwdev && hwdev->coherent_dma_mask)
351343 		dma_mask = hwdev->coherent_dma_mask;
352344 
353345 	/* do not use virt_to_phys because on ARM it doesn't return you the
354346 	 * physical address */
355
-	phys = xen_bus_to_phys(dev_addr);
347
+	phys = xen_dma_to_phys(hwdev, dev_addr);
356348 
357349 	/* Convert the size to actually allocated. */
358350 	size = 1UL << (order + XEN_PAGE_SHIFT);
359351 
352
+	if (is_vmalloc_addr(vaddr))
353
+		page = vmalloc_to_page(vaddr);
354
+	else
355
+		page = virt_to_page(vaddr);
356
+
360357 	if (!WARN_ON((dev_addr + size - 1 > dma_mask) ||
361
-		     range_straddles_page_boundary(phys, size)))
358
+		     range_straddles_page_boundary(phys, size)) &&
359
+	    TestClearPageXenRemapped(page))
362360 		xen_destroy_contiguous_region(phys, order);
363361 
364
-	xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs);
362
+	xen_free_coherent_pages(hwdev, size, vaddr, phys_to_dma(hwdev, phys),
363
+				attrs);
365364 }
366365 
367366 /*
....@@ -377,7 +376,7 @@
377376 				unsigned long attrs)
378377 {
379378 	phys_addr_t map, phys = page_to_phys(page) + offset;
380
-	dma_addr_t dev_addr = xen_phys_to_bus(phys);
379
+	dma_addr_t dev_addr = xen_phys_to_dma(dev, phys);
381380 
382381 	BUG_ON(dir == DMA_NONE);
383382 	/*
....@@ -385,41 +384,41 @@
385384 	 * we can safely return the device addr and not worry about bounce
386385 	 * buffering it.
387386 	 */
388
-	if (dma_capable(dev, dev_addr, size) &&
387
+	if (dma_capable(dev, dev_addr, size, true) &&
389388 	    !range_straddles_page_boundary(phys, size) &&
390389 		!xen_arch_need_swiotlb(dev, phys, dev_addr) &&
391
-		(swiotlb_force != SWIOTLB_FORCE)) {
392
-		/* we are not interested in the dma_addr returned by
393
-		 * xen_dma_map_page, only in the potential cache flushes executed
394
-		 * by the function. */
395
-		xen_dma_map_page(dev, page, dev_addr, offset, size, dir, attrs);
396
-		return dev_addr;
397
-	}
390
+		swiotlb_force != SWIOTLB_FORCE)
391
+		goto done;
398392 
399393 	/*
400394 	 * Oh well, have to allocate and map a bounce buffer.
401395 	 */
402396 	trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
403397 
404
-	map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir,
405
-				     attrs);
406
-	if (map == SWIOTLB_MAP_ERROR)
407
-		return XEN_SWIOTLB_ERROR_CODE;
398
+	map = swiotlb_tbl_map_single(dev, phys, size, size, dir, attrs);
399
+	if (map == (phys_addr_t)DMA_MAPPING_ERROR)
400
+		return DMA_MAPPING_ERROR;
408401 
409
-	dev_addr = xen_phys_to_bus(map);
410
-	xen_dma_map_page(dev, pfn_to_page(map >> PAGE_SHIFT),
411
-					dev_addr, map & ~PAGE_MASK, size, dir, attrs);
402
+	phys = map;
403
+	dev_addr = xen_phys_to_dma(dev, map);
412404 
413405 	/*
414406 	 * Ensure that the address returned is DMA'ble
415407 	 */
416
-	if (dma_capable(dev, dev_addr, size))
417
-		return dev_addr;
408
+	if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
409
+		swiotlb_tbl_unmap_single(dev, map, size, size, dir,
410
+				attrs | DMA_ATTR_SKIP_CPU_SYNC);
411
+		return DMA_MAPPING_ERROR;
412
+	}
418413 
419
-	attrs |= DMA_ATTR_SKIP_CPU_SYNC;
420
-	swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
421
-
422
-	return XEN_SWIOTLB_ERROR_CODE;
414
+done:
415
+	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
416
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr))))
417
+			arch_sync_dma_for_device(phys, size, dir);
418
+		else
419
+			xen_dma_sync_for_device(dev, dev_addr, size, dir);
420
+	}
421
+	return dev_addr;
423422 }
424423 
425424 /*
....@@ -430,88 +429,57 @@
430429  * After this call, reads by the cpu to the buffer are guaranteed to see
431430  * whatever the device wrote there.
432431  */
433
-static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
434
-			     size_t size, enum dma_data_direction dir,
435
-			     unsigned long attrs)
432
+static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
433
+		size_t size, enum dma_data_direction dir, unsigned long attrs)
436434 {
437
-	phys_addr_t paddr = xen_bus_to_phys(dev_addr);
435
+	phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);
438436 
439437 	BUG_ON(dir == DMA_NONE);
440438 
441
-	xen_dma_unmap_page(hwdev, dev_addr, size, dir, attrs);
442
-
443
-	/* NOTE: We use dev_addr here, not paddr! */
444
-	if (is_xen_swiotlb_buffer(dev_addr)) {
445
-		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
446
-		return;
439
+	if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
440
+		if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr))))
441
+			arch_sync_dma_for_cpu(paddr, size, dir);
442
+		else
443
+			xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir);
447444 	}
448445 
449
-	if (dir != DMA_FROM_DEVICE)
450
-		return;
451
-
452
-	/*
453
-	 * phys_to_virt doesn't work with hihgmem page but we could
454
-	 * call dma_mark_clean() with hihgmem page here. However, we
455
-	 * are fine since dma_mark_clean() is null on POWERPC. We can
456
-	 * make dma_mark_clean() take a physical address if necessary.
457
-	 */
458
-	dma_mark_clean(phys_to_virt(paddr), size);
459
-}
460
-
461
-static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
462
-			    size_t size, enum dma_data_direction dir,
463
-			    unsigned long attrs)
464
-{
465
-	xen_unmap_single(hwdev, dev_addr, size, dir, attrs);
466
-}
467
-
468
-/*
469
- * Make physical memory consistent for a single streaming mode DMA translation
470
- * after a transfer.
471
- *
472
- * If you perform a xen_swiotlb_map_page() but wish to interrogate the buffer
473
- * using the cpu, yet do not wish to teardown the dma mapping, you must
474
- * call this function before doing so.  At the next point you give the dma
475
- * address back to the card, you must first perform a
476
- * xen_swiotlb_dma_sync_for_device, and then the device again owns the buffer
477
- */
478
-static void
479
-xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
480
-			size_t size, enum dma_data_direction dir,
481
-			enum dma_sync_target target)
482
-{
483
-	phys_addr_t paddr = xen_bus_to_phys(dev_addr);
484
-
485
-	BUG_ON(dir == DMA_NONE);
486
-
487
-	if (target == SYNC_FOR_CPU)
488
-		xen_dma_sync_single_for_cpu(hwdev, dev_addr, size, dir);
489
-
490446 	/* NOTE: We use dev_addr here, not paddr! */
491
-	if (is_xen_swiotlb_buffer(dev_addr))
492
-		swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);
493
-
494
-	if (target == SYNC_FOR_DEVICE)
495
-		xen_dma_sync_single_for_device(hwdev, dev_addr, size, dir);
496
-
497
-	if (dir != DMA_FROM_DEVICE)
498
-		return;
499
-
500
-	dma_mark_clean(phys_to_virt(paddr), size);
447
+	if (is_xen_swiotlb_buffer(hwdev, dev_addr))
448
+		swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs);
501449 }
502450 
503
-void
504
-xen_swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
505
-				size_t size, enum dma_data_direction dir)
451
+static void
452
+xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
453
+		size_t size, enum dma_data_direction dir)
506454 {
507
-	xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
455
+	phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
456
+
457
+	if (!dev_is_dma_coherent(dev)) {
458
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
459
+			arch_sync_dma_for_cpu(paddr, size, dir);
460
+		else
461
+			xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
462
+	}
463
+
464
+	if (is_xen_swiotlb_buffer(dev, dma_addr))
465
+		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
508466 }
509467 
510
-void
511
-xen_swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
512
-				   size_t size, enum dma_data_direction dir)
468
+static void
469
+xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
470
+		size_t size, enum dma_data_direction dir)
513471 {
514
-	xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
472
+	phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
473
+
474
+	if (is_xen_swiotlb_buffer(dev, dma_addr))
475
+		swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
476
+
477
+	if (!dev_is_dma_coherent(dev)) {
478
+		if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
479
+			arch_sync_dma_for_device(paddr, size, dir);
480
+		else
481
+			xen_dma_sync_for_device(dev, dma_addr, size, dir);
482
+	}
515483 }
516484 
517485 /*
....@@ -519,9 +487,8 @@
519487  * concerning calls here are the same as for swiotlb_unmap_page() above.
520488  */
521489 static void
522
-xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
523
-			   int nelems, enum dma_data_direction dir,
524
-			   unsigned long attrs)
490
+xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
491
+		enum dma_data_direction dir, unsigned long attrs)
525492 {
526493 	struct scatterlist *sg;
527494 	int i;
....@@ -529,30 +496,14 @@
529496 	BUG_ON(dir == DMA_NONE);
530497 
531498 	for_each_sg(sgl, sg, nelems, i)
532
-		xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs);
499
+		xen_swiotlb_unmap_page(hwdev, sg->dma_address, sg_dma_len(sg),
500
+				dir, attrs);
533501 
534502 }
535503 
536
-/*
537
- * Map a set of buffers described by scatterlist in streaming mode for DMA.
538
- * This is the scatter-gather version of the above xen_swiotlb_map_page
539
- * interface.  Here the scatter gather list elements are each tagged with the
540
- * appropriate dma address and length.  They are obtained via
541
- * sg_dma_{address,length}(SG).
542
- *
543
- * NOTE: An implementation may be able to use a smaller number of
544
- *       DMA address/length pairs than there are SG table elements.
545
- *       (for example via virtual mapping capabilities)
546
- *       The routine returns the number of addr/length pairs actually
547
- *       used, at most nents.
548
- *
549
- * Device ownership issues as mentioned above for xen_swiotlb_map_page are the
550
- * same here.
551
- */
552504 static int
553
-xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
554
-			 int nelems, enum dma_data_direction dir,
555
-			 unsigned long attrs)
505
+xen_swiotlb_map_sg(struct device *dev, struct scatterlist *sgl, int nelems,
506
+		enum dma_data_direction dir, unsigned long attrs)
556507 {
557508 	struct scatterlist *sg;
558509 	int i;
....@@ -560,85 +511,44 @@
560511 	BUG_ON(dir == DMA_NONE);
561512 
562513 	for_each_sg(sgl, sg, nelems, i) {
563
-		phys_addr_t paddr = sg_phys(sg);
564
-		dma_addr_t dev_addr = xen_phys_to_bus(paddr);
565
-
566
-		if (swiotlb_force == SWIOTLB_FORCE ||
567
-		    xen_arch_need_swiotlb(hwdev, paddr, dev_addr) ||
568
-		    !dma_capable(hwdev, dev_addr, sg->length) ||
569
-		    range_straddles_page_boundary(paddr, sg->length)) {
570
-			phys_addr_t map = swiotlb_tbl_map_single(hwdev,
571
-								 start_dma_addr,
572
-								 sg_phys(sg),
573
-								 sg->length,
574
-								 dir, attrs);
575
-			if (map == SWIOTLB_MAP_ERROR) {
576
-				dev_warn(hwdev, "swiotlb buffer is full\n");
577
-				/* Don't panic here, we expect map_sg users
578
-				   to do proper error handling. */
579
-				attrs |= DMA_ATTR_SKIP_CPU_SYNC;
580
-				xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
581
-							   attrs);
582
-				sg_dma_len(sgl) = 0;
583
-				return 0;
584
-			}
585
-			dev_addr = xen_phys_to_bus(map);
586
-			xen_dma_map_page(hwdev, pfn_to_page(map >> PAGE_SHIFT),
587
-						dev_addr,
588
-						map & ~PAGE_MASK,
589
-						sg->length,
590
-						dir,
591
-						attrs);
592
-			sg->dma_address = dev_addr;
593
-		} else {
594
-			/* we are not interested in the dma_addr returned by
595
-			 * xen_dma_map_page, only in the potential cache flushes executed
596
-			 * by the function. */
597
-			xen_dma_map_page(hwdev, pfn_to_page(paddr >> PAGE_SHIFT),
598
-						dev_addr,
599
-						paddr & ~PAGE_MASK,
600
-						sg->length,
601
-						dir,
602
-						attrs);
603
-			sg->dma_address = dev_addr;
604
-		}
514
+		sg->dma_address = xen_swiotlb_map_page(dev, sg_page(sg),
515
+				sg->offset, sg->length, dir, attrs);
516
+		if (sg->dma_address == DMA_MAPPING_ERROR)
517
+			goto out_unmap;
605518 		sg_dma_len(sg) = sg->length;
606519 	}
520
+
607521 	return nelems;
522
+out_unmap:
523
+	xen_swiotlb_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
524
+	sg_dma_len(sgl) = 0;
525
+	return 0;
608526 }
609527 
610
-/*
611
- * Make physical memory consistent for a set of streaming mode DMA translations
612
- * after a transfer.
613
- *
614
- * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
615
- * and usage.
616
- */
617528 static void
618
-xen_swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
619
-		    int nelems, enum dma_data_direction dir,
620
-		    enum dma_sync_target target)
529
+xen_swiotlb_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
530
+			    int nelems, enum dma_data_direction dir)
621531 {
622532 	struct scatterlist *sg;
623533 	int i;
624534 
625
-	for_each_sg(sgl, sg, nelems, i)
626
-		xen_swiotlb_sync_single(hwdev, sg->dma_address,
627
-					sg_dma_len(sg), dir, target);
535
+	for_each_sg(sgl, sg, nelems, i) {
536
+		xen_swiotlb_sync_single_for_cpu(dev, sg->dma_address,
537
+				sg->length, dir);
538
+	}
628539 }
629540 
630541 static void
631
-xen_swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
632
-			    int nelems, enum dma_data_direction dir)
633
-{
634
-	xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
635
-}
636
-
637
-static void
638
-xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
542
+xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
639543 			       int nelems, enum dma_data_direction dir)
640544 {
641
-	xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
545
+	struct scatterlist *sg;
546
+	int i;
547
+
548
+	for_each_sg(sgl, sg, nelems, i) {
549
+		xen_swiotlb_sync_single_for_device(dev, sg->dma_address,
550
+				sg->length, dir);
551
+	}
642552 }
643553 
644554 /*
....@@ -650,57 +560,7 @@
650560 static int
651561 xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
652562 {
653
-	return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask;
654
-}
655
-
656
-/*
657
- * Create userspace mapping for the DMA-coherent memory.
658
- * This function should be called with the pages from the current domain only,
659
- * passing pages mapped from other domains would lead to memory corruption.
660
- */
661
-static int
662
-xen_swiotlb_dma_mmap(struct device *dev, struct vm_area_struct *vma,
663
-		     void *cpu_addr, dma_addr_t dma_addr, size_t size,
664
-		     unsigned long attrs)
665
-{
666
-#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
667
-	if (xen_get_dma_ops(dev)->mmap)
668
-		return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
669
-						    dma_addr, size, attrs);
670
-#endif
671
-	return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
672
-}
673
-
674
-/*
675
- * This function should be called with the pages from the current domain only,
676
- * passing pages mapped from other domains would lead to memory corruption.
677
- */
678
-static int
679
-xen_swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
680
-			void *cpu_addr, dma_addr_t handle, size_t size,
681
-			unsigned long attrs)
682
-{
683
-#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
684
-	if (xen_get_dma_ops(dev)->get_sgtable) {
685
-#if 0
686
-	/*
687
-	 * This check verifies that the page belongs to the current domain and
688
-	 * is not one mapped from another domain.
689
-	 * This check is for debug only, and should not go to production build
690
-	 */
691
-		unsigned long bfn = PHYS_PFN(dma_to_phys(dev, handle));
692
-		BUG_ON (!page_is_ram(bfn));
693
-#endif
694
-		return xen_get_dma_ops(dev)->get_sgtable(dev, sgt, cpu_addr,
695
-							   handle, size, attrs);
696
-	}
697
-#endif
698
-	return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size);
699
-}
700
-
701
-static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr)
702
-{
703
-	return dma_addr == XEN_SWIOTLB_ERROR_CODE;
563
+	return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask;
704564 }
705565 
706566 const struct dma_map_ops xen_swiotlb_dma_ops = {
....@@ -710,12 +570,13 @@
710570 	.sync_single_for_device = xen_swiotlb_sync_single_for_device,
711571 	.sync_sg_for_cpu = xen_swiotlb_sync_sg_for_cpu,
712572 	.sync_sg_for_device = xen_swiotlb_sync_sg_for_device,
713
-	.map_sg = xen_swiotlb_map_sg_attrs,
714
-	.unmap_sg = xen_swiotlb_unmap_sg_attrs,
573
+	.map_sg = xen_swiotlb_map_sg,
574
+	.unmap_sg = xen_swiotlb_unmap_sg,
715575 	.map_page = xen_swiotlb_map_page,
716576 	.unmap_page = xen_swiotlb_unmap_page,
717577 	.dma_supported = xen_swiotlb_dma_supported,
718
-	.mmap = xen_swiotlb_dma_mmap,
719
-	.get_sgtable = xen_swiotlb_get_sgtable,
720
-	.mapping_error	= xen_swiotlb_mapping_error,
578
+	.mmap = dma_common_mmap,
579
+	.get_sgtable = dma_common_get_sgtable,
580
+	.alloc_pages = dma_common_alloc_pages,
581
+	.free_pages = dma_common_free_pages,
721582 };