~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,17 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright 2010
3	4	* by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
4	5	*
5	6	* This code provides a IOMMU for Xen PV guests with PCI passthrough.
6		- *
7		- * This program is free software; you can redistribute it and/or modify
8		- * it under the terms of the GNU General Public License v2.0 as published by
9		- * the Free Software Foundation
10		- *
11		- * This program is distributed in the hope that it will be useful,
12		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14		- * GNU General Public License for more details.
15	7	*
16	8	* PV guests under Xen are running in an non-contiguous memory architecture.
17	9	*
..	..	@@ -30,13 +22,13 @@
30	22	* and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are
31	23	* allocated in descending order (high to low), meaning the guest might
32	24	* never get any MFN's under the 4GB mark.
33		- *
34	25	*/
35	26
36	27	#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
37	28
38		-#include <linux/bootmem.h>
	29	+#include <linux/memblock.h>
39	30	#include <linux/dma-direct.h>
	31	+#include <linux/dma-map-ops.h>
40	32	#include <linux/export.h>
41	33	#include <xen/swiotlb-xen.h>
42	34	#include <xen/page.h>
..	..	@@ -47,13 +39,12 @@
47	39	#include <asm/xen/page-coherent.h>
48	40
49	41	#include <trace/events/swiotlb.h>
	42	+#define MAX_DMA_BITS 32
50	43	/*
51	44	* Used to do a quick range check in swiotlb_tbl_unmap_single and
52	45	* swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
53	46	* API.
54	47	*/
55		-
56		-#define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0)
57	48
58	49	static char xen_io_tlb_start, xen_io_tlb_end;
59	50	static unsigned long xen_io_tlb_nslabs;
..	..	@@ -61,74 +52,60 @@
61	52	* Quick lookup value of the bus address of the IOTLB.
62	53	*/
63	54
64		-static u64 start_dma_addr;
65		-
66		-/*
67		- * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t
68		- * can be 32bit when dma_addr_t is 64bit leading to a loss in
69		- * information if the shift is done before casting to 64bit.
70		- */
71		-static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
	55	+static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr)
72	56	{
73	57	unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr));
74		- dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT;
	58	+ phys_addr_t baddr = (phys_addr_t)bfn << XEN_PAGE_SHIFT;
75	59
76		- dma \|= paddr & ~XEN_PAGE_MASK;
77		-
78		- return dma;
	60	+ baddr \|= paddr & ~XEN_PAGE_MASK;
	61	+ return baddr;
79	62	}
80	63
81		-static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr)
	64	+static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr)
	65	+{
	66	+ return phys_to_dma(dev, xen_phys_to_bus(dev, paddr));
	67	+}
	68	+
	69	+static inline phys_addr_t xen_bus_to_phys(struct device *dev,
	70	+ phys_addr_t baddr)
82	71	{
83	72	unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr));
84		- dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT;
85		- phys_addr_t paddr = dma;
86		-
87		- paddr \|= baddr & ~XEN_PAGE_MASK;
	73	+ phys_addr_t paddr = (xen_pfn << XEN_PAGE_SHIFT) \|
	74	+ (baddr & ~XEN_PAGE_MASK);
88	75
89	76	return paddr;
90	77	}
91	78
92		-static inline dma_addr_t xen_virt_to_bus(void *address)
	79	+static inline phys_addr_t xen_dma_to_phys(struct device *dev,
	80	+ dma_addr_t dma_addr)
93	81	{
94		- return xen_phys_to_bus(virt_to_phys(address));
	82	+ return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr));
95	83	}
96	84
97		-static int check_pages_physically_contiguous(unsigned long xen_pfn,
98		- unsigned int offset,
99		- size_t length)
	85	+static inline dma_addr_t xen_virt_to_bus(struct device dev, void address)
100	86	{
101		- unsigned long next_bfn;
102		- int i;
103		- int nr_pages;
104		-
105		- next_bfn = pfn_to_bfn(xen_pfn);
106		- nr_pages = (offset + length + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT;
107		-
108		- for (i = 1; i < nr_pages; i++) {
109		- if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
110		- return 0;
111		- }
112		- return 1;
	87	+ return xen_phys_to_dma(dev, virt_to_phys(address));
113	88	}
114	89
115	90	static inline int range_straddles_page_boundary(phys_addr_t p, size_t size)
116	91	{
117		- unsigned long xen_pfn = XEN_PFN_DOWN(p);
118		- unsigned int offset = p & ~XEN_PAGE_MASK;
	92	+ unsigned long next_bfn, xen_pfn = XEN_PFN_DOWN(p);
	93	+ unsigned int i, nr_pages = XEN_PFN_UP(xen_offset_in_page(p) + size);
119	94
120		- if (offset + size <= XEN_PAGE_SIZE)
121		- return 0;
122		- if (check_pages_physically_contiguous(xen_pfn, offset, size))
123		- return 0;
124		- return 1;
	95	+ next_bfn = pfn_to_bfn(xen_pfn);
	96	+
	97	+ for (i = 1; i < nr_pages; i++)
	98	+ if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
	99	+ return 1;
	100	+
	101	+ return 0;
125	102	}
126	103
127		-static int is_xen_swiotlb_buffer(dma_addr_t dma_addr)
	104	+static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
128	105	{
129		- unsigned long bfn = XEN_PFN_DOWN(dma_addr);
	106	+ unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
130	107	unsigned long xen_pfn = bfn_to_local_pfn(bfn);
131		- phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn);
	108	+ phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT;
132	109
133	110	/* If the address is outside our domain, it CAN
134	111	* have the same virtual address as another address
..	..	@@ -140,8 +117,6 @@
140	117	}
141	118	return 0;
142	119	}
143		-
144		-static int max_dma_bits = 32;
145	120
146	121	static int
147	122	xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
..	..	@@ -162,7 +137,7 @@
162	137	p + (i << IO_TLB_SHIFT),
163	138	get_order(slabs << IO_TLB_SHIFT),
164	139	dma_bits, &dma_handle);
165		- } while (rc && dma_bits++ < max_dma_bits);
	140	+ } while (rc && dma_bits++ < MAX_DMA_BITS);
166	141	if (rc)
167	142	return rc;
168	143
..	..	@@ -213,12 +188,25 @@
213	188	retry:
214	189	bytes = xen_set_nslabs(xen_io_tlb_nslabs);
215	190	order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
	191	+
	192	+ /*
	193	+ * IO TLB memory already allocated. Just use it.
	194	+ */
	195	+ if (io_tlb_start != 0) {
	196	+ xen_io_tlb_start = phys_to_virt(io_tlb_start);
	197	+ goto end;
	198	+ }
	199	+
216	200	/*
217	201	* Get IO TLB memory from any location.
218	202	*/
219		- if (early)
220		- xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
221		- else {
	203	+ if (early) {
	204	+ xen_io_tlb_start = memblock_alloc(PAGE_ALIGN(bytes),
	205	+ PAGE_SIZE);
	206	+ if (!xen_io_tlb_start)
	207	+ panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
	208	+ __func__, PAGE_ALIGN(bytes), PAGE_SIZE);
	209	+ } else {
222	210	#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
223	211	#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
224	212	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
..	..	@@ -238,7 +226,6 @@
238	226	m_ret = XEN_SWIOTLB_ENOMEM;
239	227	goto error;
240	228	}
241		- xen_io_tlb_end = xen_io_tlb_start + bytes;
242	229	/*
243	230	* And replace that memory with pages under 4GB.
244	231	*/
..	..	@@ -247,7 +234,8 @@
247	234	xen_io_tlb_nslabs);
248	235	if (rc) {
249	236	if (early)
250		- free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
	237	+ memblock_free(__pa(xen_io_tlb_start),
	238	+ PAGE_ALIGN(bytes));
251	239	else {
252	240	free_pages((unsigned long)xen_io_tlb_start, order);
253	241	xen_io_tlb_start = NULL;
..	..	@@ -255,7 +243,6 @@
255	243	m_ret = XEN_SWIOTLB_EFIXUP;
256	244	goto error;
257	245	}
258		- start_dma_addr = xen_virt_to_bus(xen_io_tlb_start);
259	246	if (early) {
260	247	if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs,
261	248	verbose))
..	..	@@ -264,6 +251,8 @@
264	251	} else
265	252	rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
266	253
	254	+end:
	255	+ xen_io_tlb_end = xen_io_tlb_start + bytes;
267	256	if (!rc)
268	257	swiotlb_set_max_segment(PAGE_SIZE);
269	258
..	..	@@ -319,12 +308,12 @@
319	308	if (hwdev && hwdev->coherent_dma_mask)
320	309	dma_mask = hwdev->coherent_dma_mask;
321	310
322		- /* At this point dma_handle is the physical address, next we are
	311	+ /* At this point dma_handle is the dma address, next we are
323	312	* going to set it to the machine address.
324	313	* Do not use virt_to_phys(ret) because on ARM it doesn't correspond
325	314	* to dma_handle. /
326		- phys = *dma_handle;
327		- dev_addr = xen_phys_to_bus(phys);
	315	+ phys = dma_to_phys(hwdev, *dma_handle);
	316	+ dev_addr = xen_phys_to_dma(hwdev, phys);
328	317	if (((dev_addr + size - 1 <= dma_mask)) &&
329	318	!range_straddles_page_boundary(phys, size))
330	319	*dma_handle = dev_addr;
..	..	@@ -334,6 +323,8 @@
334	323	xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs);
335	324	return NULL;
336	325	}
	326	+ dma_handle = phys_to_dma(hwdev, dma_handle);
	327	+ SetPageXenRemapped(virt_to_page(ret));
337	328	}
338	329	memset(ret, 0, size);
339	330	return ret;
..	..	@@ -346,22 +337,30 @@
346	337	int order = get_order(size);
347	338	phys_addr_t phys;
348	339	u64 dma_mask = DMA_BIT_MASK(32);
	340	+ struct page *page;
349	341
350	342	if (hwdev && hwdev->coherent_dma_mask)
351	343	dma_mask = hwdev->coherent_dma_mask;
352	344
353	345	/* do not use virt_to_phys because on ARM it doesn't return you the
354	346	* physical address */
355		- phys = xen_bus_to_phys(dev_addr);
	347	+ phys = xen_dma_to_phys(hwdev, dev_addr);
356	348
357	349	/* Convert the size to actually allocated. */
358	350	size = 1UL << (order + XEN_PAGE_SHIFT);
359	351
	352	+ if (is_vmalloc_addr(vaddr))
	353	+ page = vmalloc_to_page(vaddr);
	354	+ else
	355	+ page = virt_to_page(vaddr);
	356	+
360	357	if (!WARN_ON((dev_addr + size - 1 > dma_mask) \|\|
361		- range_straddles_page_boundary(phys, size)))
	358	+ range_straddles_page_boundary(phys, size)) &&
	359	+ TestClearPageXenRemapped(page))
362	360	xen_destroy_contiguous_region(phys, order);
363	361
364		- xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs);
	362	+ xen_free_coherent_pages(hwdev, size, vaddr, phys_to_dma(hwdev, phys),
	363	+ attrs);
365	364	}
366	365
367	366	/*
..	..	@@ -377,7 +376,7 @@
377	376	unsigned long attrs)
378	377	{
379	378	phys_addr_t map, phys = page_to_phys(page) + offset;
380		- dma_addr_t dev_addr = xen_phys_to_bus(phys);
	379	+ dma_addr_t dev_addr = xen_phys_to_dma(dev, phys);
381	380
382	381	BUG_ON(dir == DMA_NONE);
383	382	/*
..	..	@@ -385,41 +384,41 @@
385	384	* we can safely return the device addr and not worry about bounce
386	385	* buffering it.
387	386	*/
388		- if (dma_capable(dev, dev_addr, size) &&
	387	+ if (dma_capable(dev, dev_addr, size, true) &&
389	388	!range_straddles_page_boundary(phys, size) &&
390	389	!xen_arch_need_swiotlb(dev, phys, dev_addr) &&
391		- (swiotlb_force != SWIOTLB_FORCE)) {
392		- /* we are not interested in the dma_addr returned by
393		- * xen_dma_map_page, only in the potential cache flushes executed
394		- * by the function. */
395		- xen_dma_map_page(dev, page, dev_addr, offset, size, dir, attrs);
396		- return dev_addr;
397		- }
	390	+ swiotlb_force != SWIOTLB_FORCE)
	391	+ goto done;
398	392
399	393	/*
400	394	* Oh well, have to allocate and map a bounce buffer.
401	395	*/
402	396	trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
403	397
404		- map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir,
405		- attrs);
406		- if (map == SWIOTLB_MAP_ERROR)
407		- return XEN_SWIOTLB_ERROR_CODE;
	398	+ map = swiotlb_tbl_map_single(dev, phys, size, size, dir, attrs);
	399	+ if (map == (phys_addr_t)DMA_MAPPING_ERROR)
	400	+ return DMA_MAPPING_ERROR;
408	401
409		- dev_addr = xen_phys_to_bus(map);
410		- xen_dma_map_page(dev, pfn_to_page(map >> PAGE_SHIFT),
411		- dev_addr, map & ~PAGE_MASK, size, dir, attrs);
	402	+ phys = map;
	403	+ dev_addr = xen_phys_to_dma(dev, map);
412	404
413	405	/*
414	406	* Ensure that the address returned is DMA'ble
415	407	*/
416		- if (dma_capable(dev, dev_addr, size))
417		- return dev_addr;
	408	+ if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
	409	+ swiotlb_tbl_unmap_single(dev, map, size, size, dir,
	410	+ attrs \| DMA_ATTR_SKIP_CPU_SYNC);
	411	+ return DMA_MAPPING_ERROR;
	412	+ }
418	413
419		- attrs \|= DMA_ATTR_SKIP_CPU_SYNC;
420		- swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
421		-
422		- return XEN_SWIOTLB_ERROR_CODE;
	414	+done:
	415	+ if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
	416	+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr))))
	417	+ arch_sync_dma_for_device(phys, size, dir);
	418	+ else
	419	+ xen_dma_sync_for_device(dev, dev_addr, size, dir);
	420	+ }
	421	+ return dev_addr;
423	422	}
424	423
425	424	/*
..	..	@@ -430,88 +429,57 @@
430	429	* After this call, reads by the cpu to the buffer are guaranteed to see
431	430	* whatever the device wrote there.
432	431	*/
433		-static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
434		- size_t size, enum dma_data_direction dir,
435		- unsigned long attrs)
	432	+static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
	433	+ size_t size, enum dma_data_direction dir, unsigned long attrs)
436	434	{
437		- phys_addr_t paddr = xen_bus_to_phys(dev_addr);
	435	+ phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);
438	436
439	437	BUG_ON(dir == DMA_NONE);
440	438
441		- xen_dma_unmap_page(hwdev, dev_addr, size, dir, attrs);
442		-
443		- /* NOTE: We use dev_addr here, not paddr! */
444		- if (is_xen_swiotlb_buffer(dev_addr)) {
445		- swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
446		- return;
	439	+ if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
	440	+ if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr))))
	441	+ arch_sync_dma_for_cpu(paddr, size, dir);
	442	+ else
	443	+ xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir);
447	444	}
448	445
449		- if (dir != DMA_FROM_DEVICE)
450		- return;
451		-
452		- /*
453		- * phys_to_virt doesn't work with hihgmem page but we could
454		- * call dma_mark_clean() with hihgmem page here. However, we
455		- * are fine since dma_mark_clean() is null on POWERPC. We can
456		- * make dma_mark_clean() take a physical address if necessary.
457		- */
458		- dma_mark_clean(phys_to_virt(paddr), size);
459		-}
460		-
461		-static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
462		- size_t size, enum dma_data_direction dir,
463		- unsigned long attrs)
464		-{
465		- xen_unmap_single(hwdev, dev_addr, size, dir, attrs);
466		-}
467		-
468		-/*
469		- * Make physical memory consistent for a single streaming mode DMA translation
470		- * after a transfer.
471		- *
472		- * If you perform a xen_swiotlb_map_page() but wish to interrogate the buffer
473		- * using the cpu, yet do not wish to teardown the dma mapping, you must
474		- * call this function before doing so. At the next point you give the dma
475		- * address back to the card, you must first perform a
476		- * xen_swiotlb_dma_sync_for_device, and then the device again owns the buffer
477		- */
478		-static void
479		-xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
480		- size_t size, enum dma_data_direction dir,
481		- enum dma_sync_target target)
482		-{
483		- phys_addr_t paddr = xen_bus_to_phys(dev_addr);
484		-
485		- BUG_ON(dir == DMA_NONE);
486		-
487		- if (target == SYNC_FOR_CPU)
488		- xen_dma_sync_single_for_cpu(hwdev, dev_addr, size, dir);
489		-
490	446	/* NOTE: We use dev_addr here, not paddr! */
491		- if (is_xen_swiotlb_buffer(dev_addr))
492		- swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);
493		-
494		- if (target == SYNC_FOR_DEVICE)
495		- xen_dma_sync_single_for_device(hwdev, dev_addr, size, dir);
496		-
497		- if (dir != DMA_FROM_DEVICE)
498		- return;
499		-
500		- dma_mark_clean(phys_to_virt(paddr), size);
	447	+ if (is_xen_swiotlb_buffer(hwdev, dev_addr))
	448	+ swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs);
501	449	}
502	450
503		-void
504		-xen_swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
505		- size_t size, enum dma_data_direction dir)
	451	+static void
	452	+xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
	453	+ size_t size, enum dma_data_direction dir)
506	454	{
507		- xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
	455	+ phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
	456	+
	457	+ if (!dev_is_dma_coherent(dev)) {
	458	+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
	459	+ arch_sync_dma_for_cpu(paddr, size, dir);
	460	+ else
	461	+ xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
	462	+ }
	463	+
	464	+ if (is_xen_swiotlb_buffer(dev, dma_addr))
	465	+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
508	466	}
509	467
510		-void
511		-xen_swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
512		- size_t size, enum dma_data_direction dir)
	468	+static void
	469	+xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
	470	+ size_t size, enum dma_data_direction dir)
513	471	{
514		- xen_swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
	472	+ phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
	473	+
	474	+ if (is_xen_swiotlb_buffer(dev, dma_addr))
	475	+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
	476	+
	477	+ if (!dev_is_dma_coherent(dev)) {
	478	+ if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
	479	+ arch_sync_dma_for_device(paddr, size, dir);
	480	+ else
	481	+ xen_dma_sync_for_device(dev, dma_addr, size, dir);
	482	+ }
515	483	}
516	484
517	485	/*
..	..	@@ -519,9 +487,8 @@
519	487	* concerning calls here are the same as for swiotlb_unmap_page() above.
520	488	*/
521	489	static void
522		-xen_swiotlb_unmap_sg_attrs(struct device hwdev, struct scatterlist sgl,
523		- int nelems, enum dma_data_direction dir,
524		- unsigned long attrs)
	490	+xen_swiotlb_unmap_sg(struct device hwdev, struct scatterlist sgl, int nelems,
	491	+ enum dma_data_direction dir, unsigned long attrs)
525	492	{
526	493	struct scatterlist *sg;
527	494	int i;
..	..	@@ -529,30 +496,14 @@
529	496	BUG_ON(dir == DMA_NONE);
530	497
531	498	for_each_sg(sgl, sg, nelems, i)
532		- xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs);
	499	+ xen_swiotlb_unmap_page(hwdev, sg->dma_address, sg_dma_len(sg),
	500	+ dir, attrs);
533	501
534	502	}
535	503
536		-/*
537		- * Map a set of buffers described by scatterlist in streaming mode for DMA.
538		- * This is the scatter-gather version of the above xen_swiotlb_map_page
539		- * interface. Here the scatter gather list elements are each tagged with the
540		- * appropriate dma address and length. They are obtained via
541		- * sg_dma_{address,length}(SG).
542		- *
543		- * NOTE: An implementation may be able to use a smaller number of
544		- * DMA address/length pairs than there are SG table elements.
545		- * (for example via virtual mapping capabilities)
546		- * The routine returns the number of addr/length pairs actually
547		- * used, at most nents.
548		- *
549		- * Device ownership issues as mentioned above for xen_swiotlb_map_page are the
550		- * same here.
551		- */
552	504	static int
553		-xen_swiotlb_map_sg_attrs(struct device hwdev, struct scatterlist sgl,
554		- int nelems, enum dma_data_direction dir,
555		- unsigned long attrs)
	505	+xen_swiotlb_map_sg(struct device dev, struct scatterlist sgl, int nelems,
	506	+ enum dma_data_direction dir, unsigned long attrs)
556	507	{
557	508	struct scatterlist *sg;
558	509	int i;
..	..	@@ -560,85 +511,44 @@
560	511	BUG_ON(dir == DMA_NONE);
561	512
562	513	for_each_sg(sgl, sg, nelems, i) {
563		- phys_addr_t paddr = sg_phys(sg);
564		- dma_addr_t dev_addr = xen_phys_to_bus(paddr);
565		-
566		- if (swiotlb_force == SWIOTLB_FORCE \|\|
567		- xen_arch_need_swiotlb(hwdev, paddr, dev_addr) \|\|
568		- !dma_capable(hwdev, dev_addr, sg->length) \|\|
569		- range_straddles_page_boundary(paddr, sg->length)) {
570		- phys_addr_t map = swiotlb_tbl_map_single(hwdev,
571		- start_dma_addr,
572		- sg_phys(sg),
573		- sg->length,
574		- dir, attrs);
575		- if (map == SWIOTLB_MAP_ERROR) {
576		- dev_warn(hwdev, "swiotlb buffer is full\n");
577		- /* Don't panic here, we expect map_sg users
578		- to do proper error handling. */
579		- attrs \|= DMA_ATTR_SKIP_CPU_SYNC;
580		- xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
581		- attrs);
582		- sg_dma_len(sgl) = 0;
583		- return 0;
584		- }
585		- dev_addr = xen_phys_to_bus(map);
586		- xen_dma_map_page(hwdev, pfn_to_page(map >> PAGE_SHIFT),
587		- dev_addr,
588		- map & ~PAGE_MASK,
589		- sg->length,
590		- dir,
591		- attrs);
592		- sg->dma_address = dev_addr;
593		- } else {
594		- /* we are not interested in the dma_addr returned by
595		- * xen_dma_map_page, only in the potential cache flushes executed
596		- * by the function. */
597		- xen_dma_map_page(hwdev, pfn_to_page(paddr >> PAGE_SHIFT),
598		- dev_addr,
599		- paddr & ~PAGE_MASK,
600		- sg->length,
601		- dir,
602		- attrs);
603		- sg->dma_address = dev_addr;
604		- }
	514	+ sg->dma_address = xen_swiotlb_map_page(dev, sg_page(sg),
	515	+ sg->offset, sg->length, dir, attrs);
	516	+ if (sg->dma_address == DMA_MAPPING_ERROR)
	517	+ goto out_unmap;
605	518	sg_dma_len(sg) = sg->length;
606	519	}
	520	+
607	521	return nelems;
	522	+out_unmap:
	523	+ xen_swiotlb_unmap_sg(dev, sgl, i, dir, attrs \| DMA_ATTR_SKIP_CPU_SYNC);
	524	+ sg_dma_len(sgl) = 0;
	525	+ return 0;
608	526	}
609	527
610		-/*
611		- * Make physical memory consistent for a set of streaming mode DMA translations
612		- * after a transfer.
613		- *
614		- * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
615		- * and usage.
616		- */
617	528	static void
618		-xen_swiotlb_sync_sg(struct device hwdev, struct scatterlist sgl,
619		- int nelems, enum dma_data_direction dir,
620		- enum dma_sync_target target)
	529	+xen_swiotlb_sync_sg_for_cpu(struct device dev, struct scatterlist sgl,
	530	+ int nelems, enum dma_data_direction dir)
621	531	{
622	532	struct scatterlist *sg;
623	533	int i;
624	534
625		- for_each_sg(sgl, sg, nelems, i)
626		- xen_swiotlb_sync_single(hwdev, sg->dma_address,
627		- sg_dma_len(sg), dir, target);
	535	+ for_each_sg(sgl, sg, nelems, i) {
	536	+ xen_swiotlb_sync_single_for_cpu(dev, sg->dma_address,
	537	+ sg->length, dir);
	538	+ }
628	539	}
629	540
630	541	static void
631		-xen_swiotlb_sync_sg_for_cpu(struct device hwdev, struct scatterlist sg,
632		- int nelems, enum dma_data_direction dir)
633		-{
634		- xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
635		-}
636		-
637		-static void
638		-xen_swiotlb_sync_sg_for_device(struct device hwdev, struct scatterlist sg,
	542	+xen_swiotlb_sync_sg_for_device(struct device dev, struct scatterlist sgl,
639	543	int nelems, enum dma_data_direction dir)
640	544	{
641		- xen_swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
	545	+ struct scatterlist *sg;
	546	+ int i;
	547	+
	548	+ for_each_sg(sgl, sg, nelems, i) {
	549	+ xen_swiotlb_sync_single_for_device(dev, sg->dma_address,
	550	+ sg->length, dir);
	551	+ }
642	552	}
643	553
644	554	/*
..	..	@@ -650,57 +560,7 @@
650	560	static int
651	561	xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
652	562	{
653		- return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask;
654		-}
655		-
656		-/*
657		- * Create userspace mapping for the DMA-coherent memory.
658		- * This function should be called with the pages from the current domain only,
659		- * passing pages mapped from other domains would lead to memory corruption.
660		- */
661		-static int
662		-xen_swiotlb_dma_mmap(struct device dev, struct vm_area_struct vma,
663		- void *cpu_addr, dma_addr_t dma_addr, size_t size,
664		- unsigned long attrs)
665		-{
666		-#if defined(CONFIG_ARM) \|\| defined(CONFIG_ARM64)
667		- if (xen_get_dma_ops(dev)->mmap)
668		- return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
669		- dma_addr, size, attrs);
670		-#endif
671		- return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
672		-}
673		-
674		-/*
675		- * This function should be called with the pages from the current domain only,
676		- * passing pages mapped from other domains would lead to memory corruption.
677		- */
678		-static int
679		-xen_swiotlb_get_sgtable(struct device dev, struct sg_table sgt,
680		- void *cpu_addr, dma_addr_t handle, size_t size,
681		- unsigned long attrs)
682		-{
683		-#if defined(CONFIG_ARM) \|\| defined(CONFIG_ARM64)
684		- if (xen_get_dma_ops(dev)->get_sgtable) {
685		-#if 0
686		- /*
687		- * This check verifies that the page belongs to the current domain and
688		- * is not one mapped from another domain.
689		- * This check is for debug only, and should not go to production build
690		- */
691		- unsigned long bfn = PHYS_PFN(dma_to_phys(dev, handle));
692		- BUG_ON (!page_is_ram(bfn));
693		-#endif
694		- return xen_get_dma_ops(dev)->get_sgtable(dev, sgt, cpu_addr,
695		- handle, size, attrs);
696		- }
697		-#endif
698		- return dma_common_get_sgtable(dev, sgt, cpu_addr, handle, size);
699		-}
700		-
701		-static int xen_swiotlb_mapping_error(struct device *dev, dma_addr_t dma_addr)
702		-{
703		- return dma_addr == XEN_SWIOTLB_ERROR_CODE;
	563	+ return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask;
704	564	}
705	565
706	566	const struct dma_map_ops xen_swiotlb_dma_ops = {
..	..	@@ -710,12 +570,13 @@
710	570	.sync_single_for_device = xen_swiotlb_sync_single_for_device,
711	571	.sync_sg_for_cpu = xen_swiotlb_sync_sg_for_cpu,
712	572	.sync_sg_for_device = xen_swiotlb_sync_sg_for_device,
713		- .map_sg = xen_swiotlb_map_sg_attrs,
714		- .unmap_sg = xen_swiotlb_unmap_sg_attrs,
	573	+ .map_sg = xen_swiotlb_map_sg,
	574	+ .unmap_sg = xen_swiotlb_unmap_sg,
715	575	.map_page = xen_swiotlb_map_page,
716	576	.unmap_page = xen_swiotlb_unmap_page,
717	577	.dma_supported = xen_swiotlb_dma_supported,
718		- .mmap = xen_swiotlb_dma_mmap,
719		- .get_sgtable = xen_swiotlb_get_sgtable,
720		- .mapping_error = xen_swiotlb_mapping_error,
	578	+ .mmap = dma_common_mmap,
	579	+ .get_sgtable = dma_common_get_sgtable,
	580	+ .alloc_pages = dma_common_alloc_pages,
	581	+ .free_pages = dma_common_free_pages,
721	582	};