forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/arch/arm64/mm/dma-mapping.c
....@@ -1,1139 +1,72 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
2
- * SWIOTLB-based DMA API implementation
3
- *
43  * Copyright (C) 2012 ARM Ltd.
54  * Author: Catalin Marinas <catalin.marinas@arm.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify
8
- * it under the terms of the GNU General Public License version 2 as
9
- * published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope that it will be useful,
12
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
- * GNU General Public License for more details.
15
- *
16
- * You should have received a copy of the GNU General Public License
17
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
185  */
196 
207 #include <linux/gfp.h>
21
-#include <linux/acpi.h>
22
-#include <linux/bootmem.h>
238 #include <linux/cache.h>
24
-#include <linux/export.h>
25
-#include <linux/slab.h>
26
-#include <linux/genalloc.h>
27
-#include <linux/dma-direct.h>
28
-#include <linux/dma-contiguous.h>
29
-#include <linux/mm.h>
30
-#include <linux/iommu.h>
31
-#include <linux/vmalloc.h>
32
-#include <linux/swiotlb.h>
33
-#include <linux/dma-removed.h>
34
-#include <linux/pci.h>
35
-#include <linux/io.h>
9
+#include <linux/dma-map-ops.h>
10
+#include <linux/dma-iommu.h>
11
+#include <xen/xen.h>
12
+#include <xen/swiotlb-xen.h>
13
+#include <trace/hooks/iommu.h>
14
+#include <trace/hooks/dma_noalias.h>
3615 
3716 #include <asm/cacheflush.h>
38
-#include <asm/tlbflush.h>
39
-#include <asm/dma-iommu.h>
40
-#include <linux/of_address.h>
41
-#include <linux/dma-mapping-fast.h>
4217 
43
-static int swiotlb __ro_after_init;
44
-
45
-static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot,
46
-				 bool coherent)
18
+void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
19
+		enum dma_data_direction dir)
4720 {
48
-	if (!coherent || (attrs & DMA_ATTR_WRITE_COMBINE))
49
-		return pgprot_writecombine(prot);
50
-	return prot;
21
+	__dma_map_area(phys_to_virt(paddr), size, dir);
5122 }
5223 
53
-static bool is_dma_coherent(struct device *dev, unsigned long attrs)
24
+void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
25
+		enum dma_data_direction dir)
5426 {
55
-
56
-	if (attrs & DMA_ATTR_FORCE_COHERENT)
57
-		return true;
58
-	else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
59
-		return false;
60
-	else if (is_device_dma_coherent(dev))
61
-		return true;
62
-	else
63
-		return false;
64
-}
65
-static struct gen_pool *atomic_pool __ro_after_init;
66
-
67
-#define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
68
-static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
69
-
70
-static int __init early_coherent_pool(char *p)
71
-{
72
-	atomic_pool_size = memparse(p, &p);
73
-	return 0;
74
-}
75
-early_param("coherent_pool", early_coherent_pool);
76
-
77
-static void *__alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
78
-{
79
-	unsigned long val;
80
-	void *ptr = NULL;
81
-
82
-	if (!atomic_pool) {
83
-		WARN(1, "coherent pool not initialised!\n");
84
-		return NULL;
85
-	}
86
-
87
-	val = gen_pool_alloc(atomic_pool, size);
88
-	if (val) {
89
-		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
90
-
91
-		*ret_page = phys_to_page(phys);
92
-		ptr = (void *)val;
93
-		memset(ptr, 0, size);
94
-	}
95
-
96
-	return ptr;
27
+	__dma_unmap_area(phys_to_virt(paddr), size, dir);
9728 }
9829 
99
-static bool __in_atomic_pool(void *start, size_t size)
30
+void arch_dma_prep_coherent(struct page *page, size_t size)
10031 {
101
-	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
32
+	__dma_flush_area(page_address(page), size);
10233 }
103
-
104
-static int __free_from_pool(void *start, size_t size)
105
-{
106
-	if (!__in_atomic_pool(start, size))
107
-		return 0;
108
-
109
-	gen_pool_free(atomic_pool, (unsigned long)start, size);
110
-
111
-	return 1;
112
-}
113
-
114
-static void *__dma_alloc(struct device *dev, size_t size,
115
-			 dma_addr_t *dma_handle, gfp_t flags,
116
-			 unsigned long attrs)
117
-{
118
-	struct page *page;
119
-	void *ptr, *coherent_ptr;
120
-	bool coherent = is_dma_coherent(dev, attrs);
121
-	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);
122
-
123
-	size = PAGE_ALIGN(size);
124
-
125
-	if (!coherent && !gfpflags_allow_blocking(flags)) {
126
-		struct page *page = NULL;
127
-		void *addr = __alloc_from_pool(size, &page, flags);
128
-
129
-		if (addr)
130
-			*dma_handle = phys_to_dma(dev, page_to_phys(page));
131
-
132
-		return addr;
133
-	}
134
-
135
-	ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs);
136
-
137
-	if (!ptr)
138
-		goto no_mem;
139
-
140
-	/* no need for non-cacheable mapping if coherent */
141
-	if (coherent)
142
-		return ptr;
143
-
144
-	/* remove any dirty cache lines on the kernel alias */
145
-	__dma_flush_area(ptr, size);
146
-
147
-	/* create a coherent mapping */
148
-	page = virt_to_page(ptr);
149
-	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
150
-						   prot, __builtin_return_address(0));
151
-	if (!coherent_ptr)
152
-		goto no_map;
153
-
154
-	return coherent_ptr;
155
-
156
-no_map:
157
-	swiotlb_free(dev, size, ptr, *dma_handle, attrs);
158
-no_mem:
159
-	return NULL;
160
-}
161
-
162
-static void __dma_free(struct device *dev, size_t size,
163
-		       void *vaddr, dma_addr_t dma_handle,
164
-		       unsigned long attrs)
165
-{
166
-	void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
167
-
168
-	size = PAGE_ALIGN(size);
169
-
170
-	if (!is_device_dma_coherent(dev)) {
171
-		if (__free_from_pool(vaddr, size))
172
-			return;
173
-		vunmap(vaddr);
174
-	}
175
-	swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs);
176
-}
177
-
178
-static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
179
-				     unsigned long offset, size_t size,
180
-				     enum dma_data_direction dir,
181
-				     unsigned long attrs)
182
-{
183
-	dma_addr_t dev_addr;
184
-
185
-	dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
186
-	if (!is_dma_coherent(dev, attrs) &&
187
-	    (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
188
-		__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
189
-
190
-	return dev_addr;
191
-}
192
-
193
-
194
-static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
195
-				 size_t size, enum dma_data_direction dir,
196
-				 unsigned long attrs)
197
-{
198
-	if (!is_dma_coherent(dev, attrs) &&
199
-	    (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
200
-		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
201
-	swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
202
-}
203
-
204
-static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
205
-				  int nelems, enum dma_data_direction dir,
206
-				  unsigned long attrs)
207
-{
208
-	struct scatterlist *sg;
209
-	int i, ret;
210
-
211
-	ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
212
-	if (!is_dma_coherent(dev, attrs) &&
213
-	    (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
214
-		for_each_sg(sgl, sg, ret, i)
215
-			__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
216
-				       sg->length, dir);
217
-
218
-	return ret;
219
-}
220
-
221
-static void __swiotlb_unmap_sg_attrs(struct device *dev,
222
-				     struct scatterlist *sgl, int nelems,
223
-				     enum dma_data_direction dir,
224
-				     unsigned long attrs)
225
-{
226
-	struct scatterlist *sg;
227
-	int i;
228
-
229
-	if (!is_dma_coherent(dev, attrs) &&
230
-	    (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
231
-		for_each_sg(sgl, sg, nelems, i)
232
-			__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
233
-					 sg->length, dir);
234
-	swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
235
-}
236
-
237
-static void __swiotlb_sync_single_for_cpu(struct device *dev,
238
-					  dma_addr_t dev_addr, size_t size,
239
-					  enum dma_data_direction dir)
240
-{
241
-	if (!is_device_dma_coherent(dev))
242
-		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
243
-	swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
244
-}
245
-
246
-static void __swiotlb_sync_single_for_device(struct device *dev,
247
-					     dma_addr_t dev_addr, size_t size,
248
-					     enum dma_data_direction dir)
249
-{
250
-	swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
251
-	if (!is_device_dma_coherent(dev))
252
-		__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
253
-}
254
-
255
-static void __swiotlb_sync_sg_for_cpu(struct device *dev,
256
-				      struct scatterlist *sgl, int nelems,
257
-				      enum dma_data_direction dir)
258
-{
259
-	struct scatterlist *sg;
260
-	int i;
261
-
262
-	if (!is_device_dma_coherent(dev))
263
-		for_each_sg(sgl, sg, nelems, i)
264
-			__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
265
-					 sg->length, dir);
266
-	swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
267
-}
268
-
269
-static void __swiotlb_sync_sg_for_device(struct device *dev,
270
-					 struct scatterlist *sgl, int nelems,
271
-					 enum dma_data_direction dir)
272
-{
273
-	struct scatterlist *sg;
274
-	int i;
275
-
276
-	swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
277
-	if (!is_device_dma_coherent(dev))
278
-		for_each_sg(sgl, sg, nelems, i)
279
-			__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
280
-				       sg->length, dir);
281
-}
282
-
283
-static int __swiotlb_mmap_pfn(struct vm_area_struct *vma,
284
-			      unsigned long pfn, size_t size)
285
-{
286
-	int ret = -ENXIO;
287
-	unsigned long nr_vma_pages = vma_pages(vma);
288
-	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
289
-	unsigned long off = vma->vm_pgoff;
290
-
291
-	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
292
-		ret = remap_pfn_range(vma, vma->vm_start,
293
-				      pfn + off,
294
-				      vma->vm_end - vma->vm_start,
295
-				      vma->vm_page_prot);
296
-	}
297
-
298
-	return ret;
299
-}
300
-
301
-static int __swiotlb_mmap(struct device *dev,
302
-			  struct vm_area_struct *vma,
303
-			  void *cpu_addr, dma_addr_t dma_addr, size_t size,
304
-			  unsigned long attrs)
305
-{
306
-	int ret = -ENXIO;
307
-	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
308
-
309
-	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
310
-			is_dma_coherent(dev, attrs));
311
-
312
-	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
313
-		return ret;
314
-
315
-	return __swiotlb_mmap_pfn(vma, pfn, size);
316
-}
317
-
318
-static int __swiotlb_get_sgtable_page(struct sg_table *sgt,
319
-				      struct page *page, size_t size)
320
-{
321
-	int ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
322
-
323
-	if (!ret)
324
-		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
325
-
326
-	return ret;
327
-}
328
-
329
-static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt,
330
-				 void *cpu_addr, dma_addr_t handle, size_t size,
331
-				 unsigned long attrs)
332
-{
333
-	struct page *page = phys_to_page(dma_to_phys(dev, handle));
334
-
335
-	return __swiotlb_get_sgtable_page(sgt, page, size);
336
-}
337
-
338
-static int __swiotlb_dma_supported(struct device *hwdev, u64 mask)
339
-{
340
-	if (swiotlb)
341
-		return swiotlb_dma_supported(hwdev, mask);
342
-	return 1;
343
-}
344
-
345
-static void *arm64_dma_remap(struct device *dev, void *cpu_addr,
346
-			dma_addr_t handle, size_t size,
347
-			unsigned long attrs)
348
-{
349
-	struct page *page = phys_to_page(dma_to_phys(dev, handle));
350
-	bool coherent = is_device_dma_coherent(dev);
351
-	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
352
-	unsigned long offset = handle & ~PAGE_MASK;
353
-	struct vm_struct *area;
354
-	unsigned long addr;
355
-
356
-	size = PAGE_ALIGN(size + offset);
357
-
358
-	/*
359
-	 * DMA allocation can be mapped to user space, so lets
360
-	 * set VM_USERMAP flags too.
361
-	 */
362
-	area = get_vm_area(size, VM_USERMAP);
363
-	if (!area)
364
-		return NULL;
365
-
366
-	addr = (unsigned long)area->addr;
367
-	area->phys_addr = __pfn_to_phys(page_to_pfn(page));
368
-
369
-	if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
370
-		vunmap((void *)addr);
371
-		return NULL;
372
-	}
373
-	return (void *)addr + offset;
374
-}
375
-
376
-static void arm64_dma_unremap(struct device *dev, void *remapped_addr,
377
-				size_t size)
378
-{
379
-	struct vm_struct *area;
380
-
381
-	size = PAGE_ALIGN(size);
382
-	remapped_addr = (void *)((unsigned long)remapped_addr & PAGE_MASK);
383
-
384
-	area = find_vm_area(remapped_addr);
385
-	if (!area) {
386
-		WARN(1, "trying to free invalid coherent area: %pK\n",
387
-			remapped_addr);
388
-		return;
389
-	}
390
-	vunmap(remapped_addr);
391
-	flush_tlb_kernel_range((unsigned long)remapped_addr,
392
-			(unsigned long)(remapped_addr + size));
393
-}
394
-
395
-static int __swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t addr)
396
-{
397
-	if (swiotlb)
398
-		return swiotlb_dma_mapping_error(hwdev, addr);
399
-	return 0;
400
-}
401
-
402
-static const struct dma_map_ops arm64_swiotlb_dma_ops = {
403
-	.alloc = __dma_alloc,
404
-	.free = __dma_free,
405
-	.mmap = __swiotlb_mmap,
406
-	.get_sgtable = __swiotlb_get_sgtable,
407
-	.map_page = __swiotlb_map_page,
408
-	.unmap_page = __swiotlb_unmap_page,
409
-	.map_sg = __swiotlb_map_sg_attrs,
410
-	.unmap_sg = __swiotlb_unmap_sg_attrs,
411
-	.sync_single_for_cpu = __swiotlb_sync_single_for_cpu,
412
-	.sync_single_for_device = __swiotlb_sync_single_for_device,
413
-	.sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu,
414
-	.sync_sg_for_device = __swiotlb_sync_sg_for_device,
415
-	.dma_supported = __swiotlb_dma_supported,
416
-	.mapping_error = __swiotlb_dma_mapping_error,
417
-	.remap = arm64_dma_remap,
418
-	.unremap = arm64_dma_unremap,
419
-};
420
-
421
-static int __init atomic_pool_init(void)
422
-{
423
-	pgprot_t prot = __pgprot(PROT_NORMAL_NC);
424
-	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
425
-	struct page *page;
426
-	void *addr;
427
-	unsigned int pool_size_order = get_order(atomic_pool_size);
428
-
429
-	if (dev_get_cma_area(NULL))
430
-		page = dma_alloc_from_contiguous(NULL, nr_pages,
431
-						 pool_size_order, false);
432
-	else
433
-		page = alloc_pages(GFP_DMA32, pool_size_order);
434
-
435
-	if (page) {
436
-		int ret;
437
-		void *page_addr = page_address(page);
438
-
439
-		memset(page_addr, 0, atomic_pool_size);
440
-		__dma_flush_area(page_addr, atomic_pool_size);
441
-
442
-		atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
443
-		if (!atomic_pool)
444
-			goto free_page;
445
-
446
-		addr = dma_common_contiguous_remap(page, atomic_pool_size,
447
-					VM_USERMAP, prot, atomic_pool_init);
448
-
449
-		if (!addr)
450
-			goto destroy_genpool;
451
-
452
-		ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
453
-					page_to_phys(page),
454
-					atomic_pool_size, -1);
455
-		if (ret)
456
-			goto remove_mapping;
457
-
458
-		gen_pool_set_algo(atomic_pool,
459
-				  gen_pool_first_fit_order_align,
460
-				  NULL);
461
-
462
-		pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
463
-			atomic_pool_size / 1024);
464
-		return 0;
465
-	}
466
-	goto out;
467
-
468
-remove_mapping:
469
-	dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP, false);
470
-destroy_genpool:
471
-	gen_pool_destroy(atomic_pool);
472
-	atomic_pool = NULL;
473
-free_page:
474
-	if (!dma_release_from_contiguous(NULL, page, nr_pages))
475
-		__free_pages(page, pool_size_order);
476
-out:
477
-	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
478
-		atomic_pool_size / 1024);
479
-	return -ENOMEM;
480
-}
481
-
482
-/********************************************
483
- * The following APIs are for dummy DMA ops *
484
- ********************************************/
485
-
486
-static void *__dummy_alloc(struct device *dev, size_t size,
487
-			   dma_addr_t *dma_handle, gfp_t flags,
488
-			   unsigned long attrs)
489
-{
490
-	WARN(1, "dma alloc failure, device may be missing a call to arch_setup_dma_ops");
491
-	return NULL;
492
-}
493
-
494
-static void __dummy_free(struct device *dev, size_t size,
495
-			 void *vaddr, dma_addr_t dma_handle,
496
-			 unsigned long attrs)
497
-{
498
-}
499
-
500
-static int __dummy_mmap(struct device *dev,
501
-			struct vm_area_struct *vma,
502
-			void *cpu_addr, dma_addr_t dma_addr, size_t size,
503
-			unsigned long attrs)
504
-{
505
-	return -ENXIO;
506
-}
507
-
508
-static dma_addr_t __dummy_map_page(struct device *dev, struct page *page,
509
-				   unsigned long offset, size_t size,
510
-				   enum dma_data_direction dir,
511
-				   unsigned long attrs)
512
-{
513
-	return 0;
514
-}
515
-
516
-static void __dummy_unmap_page(struct device *dev, dma_addr_t dev_addr,
517
-			       size_t size, enum dma_data_direction dir,
518
-			       unsigned long attrs)
519
-{
520
-}
521
-
522
-static int __dummy_map_sg(struct device *dev, struct scatterlist *sgl,
523
-			  int nelems, enum dma_data_direction dir,
524
-			  unsigned long attrs)
525
-{
526
-	return 0;
527
-}
528
-
529
-static void __dummy_unmap_sg(struct device *dev,
530
-			     struct scatterlist *sgl, int nelems,
531
-			     enum dma_data_direction dir,
532
-			     unsigned long attrs)
533
-{
534
-}
535
-
536
-static void __dummy_sync_single(struct device *dev,
537
-				dma_addr_t dev_addr, size_t size,
538
-				enum dma_data_direction dir)
539
-{
540
-}
541
-
542
-static void __dummy_sync_sg(struct device *dev,
543
-			    struct scatterlist *sgl, int nelems,
544
-			    enum dma_data_direction dir)
545
-{
546
-}
547
-
548
-static int __dummy_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
549
-{
550
-	return 1;
551
-}
552
-
553
-static int __dummy_dma_supported(struct device *hwdev, u64 mask)
554
-{
555
-	return 0;
556
-}
557
-
558
-const struct dma_map_ops dummy_dma_ops = {
559
-	.alloc                  = __dummy_alloc,
560
-	.free                   = __dummy_free,
561
-	.mmap                   = __dummy_mmap,
562
-	.map_page               = __dummy_map_page,
563
-	.unmap_page             = __dummy_unmap_page,
564
-	.map_sg                 = __dummy_map_sg,
565
-	.unmap_sg               = __dummy_unmap_sg,
566
-	.sync_single_for_cpu    = __dummy_sync_single,
567
-	.sync_single_for_device = __dummy_sync_single,
568
-	.sync_sg_for_cpu        = __dummy_sync_sg,
569
-	.sync_sg_for_device     = __dummy_sync_sg,
570
-	.mapping_error          = __dummy_mapping_error,
571
-	.dma_supported          = __dummy_dma_supported,
572
-};
573
-EXPORT_SYMBOL(dummy_dma_ops);
574
-
575
-static int __init arm64_dma_init(void)
576
-{
577
-	if (swiotlb_force == SWIOTLB_FORCE ||
578
-	    max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT))
579
-		swiotlb = 1;
580
-
581
-	WARN_TAINT(ARCH_DMA_MINALIGN < cache_line_size(),
582
-		   TAINT_CPU_OUT_OF_SPEC,
583
-		   "ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)",
584
-		   ARCH_DMA_MINALIGN, cache_line_size());
585
-
586
-	return atomic_pool_init();
587
-}
588
-arch_initcall(arm64_dma_init);
58934 
59035 #ifdef CONFIG_IOMMU_DMA
591
-#include <linux/dma-iommu.h>
592
-#include <linux/platform_device.h>
593
-#include <linux/amba/bus.h>
594
-
595
-/* Thankfully, all cache ops are by VA so we can ignore phys here */
596
-static void flush_page(struct device *dev, const void *virt, phys_addr_t phys)
597
-{
598
-	__dma_flush_area(virt, PAGE_SIZE);
599
-}
600
-
601
-static void *__iommu_alloc_attrs(struct device *dev, size_t size,
602
-				 dma_addr_t *handle, gfp_t gfp,
603
-				 unsigned long attrs)
604
-{
605
-	bool coherent = is_dma_coherent(dev, attrs);
606
-	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
607
-	size_t iosize = size;
608
-	void *addr;
609
-
610
-	if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n"))
611
-		return NULL;
612
-
613
-	size = PAGE_ALIGN(size);
614
-
615
-	/*
616
-	 * Some drivers rely on this, and we probably don't want the
617
-	 * possibility of stale kernel data being read by devices anyway.
618
-	 */
619
-	if (!(attrs & DMA_ATTR_SKIP_ZEROING))
620
-		gfp |= __GFP_ZERO;
621
-
622
-	if (!gfpflags_allow_blocking(gfp)) {
623
-		struct page *page;
624
-		/*
625
-		 * In atomic context we can't remap anything, so we'll only
626
-		 * get the virtually contiguous buffer we need by way of a
627
-		 * physically contiguous allocation.
628
-		 */
629
-		if (coherent) {
630
-			page = alloc_pages(gfp, get_order(size));
631
-			addr = page ? page_address(page) : NULL;
632
-		} else {
633
-			addr = __alloc_from_pool(size, &page, gfp);
634
-		}
635
-		if (!addr)
636
-			return NULL;
637
-
638
-		*handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);
639
-		if (iommu_dma_mapping_error(dev, *handle)) {
640
-			if (coherent)
641
-				__free_pages(page, get_order(size));
642
-			else
643
-				__free_from_pool(addr, size);
644
-			addr = NULL;
645
-		}
646
-	} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
647
-		pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
648
-		struct page *page;
649
-
650
-		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
651
-					get_order(size), gfp & __GFP_NOWARN);
652
-		if (!page)
653
-			return NULL;
654
-
655
-		*handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);
656
-		if (iommu_dma_mapping_error(dev, *handle)) {
657
-			dma_release_from_contiguous(dev, page,
658
-						    size >> PAGE_SHIFT);
659
-			return NULL;
660
-		}
661
-		addr = dma_common_contiguous_remap(page, size, VM_USERMAP,
662
-						   prot,
663
-						   __builtin_return_address(0));
664
-		if (addr) {
665
-			if (!coherent)
666
-				__dma_flush_area(page_to_virt(page), iosize);
667
-			memset(addr, 0, size);
668
-		} else {
669
-			iommu_dma_unmap_page(dev, *handle, iosize, 0, attrs);
670
-			dma_release_from_contiguous(dev, page,
671
-						    size >> PAGE_SHIFT);
672
-		}
673
-	} else {
674
-		pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
675
-		struct page **pages;
676
-
677
-		pages = iommu_dma_alloc(dev, iosize, gfp, attrs, ioprot,
678
-					handle, flush_page);
679
-		if (!pages)
680
-			return NULL;
681
-
682
-		addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,
683
-					      __builtin_return_address(0));
684
-		if (!addr)
685
-			iommu_dma_free(dev, pages, iosize, handle);
686
-	}
687
-	return addr;
688
-}
689
-
690
-static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
691
-			       dma_addr_t handle, unsigned long attrs)
692
-{
693
-	size_t iosize = size;
694
-
695
-	size = PAGE_ALIGN(size);
696
-	/*
697
-	 * @cpu_addr will be one of 4 things depending on how it was allocated:
698
-	 * - A remapped array of pages for contiguous allocations.
699
-	 * - A remapped array of pages from iommu_dma_alloc(), for all
700
-	 *   non-atomic allocations.
701
-	 * - A non-cacheable alias from the atomic pool, for atomic
702
-	 *   allocations by non-coherent devices.
703
-	 * - A normal lowmem address, for atomic allocations by
704
-	 *   coherent devices.
705
-	 * Hence how dodgy the below logic looks...
706
-	 */
707
-	if (__in_atomic_pool(cpu_addr, size)) {
708
-		iommu_dma_unmap_page(dev, handle, iosize, 0, 0);
709
-		__free_from_pool(cpu_addr, size);
710
-	} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
711
-		struct page *page = vmalloc_to_page(cpu_addr);
712
-		iommu_dma_unmap_page(dev, handle, iosize, 0, attrs);
713
-		dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
714
-		dma_common_free_remap(cpu_addr, size, VM_USERMAP, false);
715
-	} else if (is_vmalloc_addr(cpu_addr)) {
716
-		struct vm_struct *area = find_vm_area(cpu_addr);
717
-
718
-		if (WARN_ON(!area || !area->pages))
719
-			return;
720
-		iommu_dma_free(dev, area->pages, iosize, &handle);
721
-		dma_common_free_remap(cpu_addr, size, VM_USERMAP, false);
722
-	} else {
723
-		iommu_dma_unmap_page(dev, handle, iosize, 0, 0);
724
-		__free_pages(virt_to_page(cpu_addr), get_order(size));
725
-	}
726
-}
727
-
728
-static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
729
-			      void *cpu_addr, dma_addr_t dma_addr, size_t size,
730
-			      unsigned long attrs)
731
-{
732
-	struct vm_struct *area;
733
-	int ret;
734
-	unsigned long pfn = 0;
735
-
736
-	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
737
-					     is_dma_coherent(dev, attrs));
738
-
739
-	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
740
-		return ret;
741
-
742
-	area = find_vm_area(cpu_addr);
743
-
744
-	if (area && area->pages)
745
-		return iommu_dma_mmap(area->pages, size, vma);
746
-	else if (!is_vmalloc_addr(cpu_addr))
747
-		pfn = page_to_pfn(virt_to_page(cpu_addr));
748
-	else if (is_vmalloc_addr(cpu_addr))
749
-		/*
750
-		 * DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations are
751
-		 * always remapped, hence in the vmalloc space.
752
-		 */
753
-		pfn = vmalloc_to_pfn(cpu_addr);
754
-
755
-	if (pfn)
756
-		return __swiotlb_mmap_pfn(vma, pfn, size);
757
-
758
-	return -ENXIO;
759
-}
760
-
761
-static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
762
-			       void *cpu_addr, dma_addr_t dma_addr,
763
-			       size_t size, unsigned long attrs)
764
-{
765
-	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
766
-	struct page *page = NULL;
767
-	struct vm_struct *area = find_vm_area(cpu_addr);
768
-
769
-	if (area && area->pages)
770
-		return sg_alloc_table_from_pages(sgt, area->pages, count, 0,
771
-					size, GFP_KERNEL);
772
-	else if (!is_vmalloc_addr(cpu_addr))
773
-		page = virt_to_page(cpu_addr);
774
-	else if (is_vmalloc_addr(cpu_addr))
775
-		/*
776
-		 * DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations
777
-		 * are always remapped, hence in the vmalloc space.
778
-		 */
779
-		page = vmalloc_to_page(cpu_addr);
780
-
781
-	if (page)
782
-		return __swiotlb_get_sgtable_page(sgt, page, size);
783
-	return -ENXIO;
784
-}
785
-
786
-static void __iommu_sync_single_for_cpu(struct device *dev,
787
-					dma_addr_t dev_addr, size_t size,
788
-					enum dma_data_direction dir)
789
-{
790
-	phys_addr_t phys;
791
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
792
-
793
-	if (!domain || iommu_is_iova_coherent(domain, dev_addr))
794
-		return;
795
-	if (is_device_dma_coherent(dev))
796
-		return;
797
-	phys = iommu_iova_to_phys(domain, dev_addr);
798
-	__dma_unmap_area(phys_to_virt(phys), size, dir);
799
-}
800
-
801
-static void __iommu_sync_single_for_device(struct device *dev,
802
-					   dma_addr_t dev_addr, size_t size,
803
-					   enum dma_data_direction dir)
804
-{
805
-	phys_addr_t phys;
806
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
807
-
808
-	if (!domain || iommu_is_iova_coherent(domain, dev_addr))
809
-		return;
810
-	if (is_device_dma_coherent(dev))
811
-		return;
812
-	phys = iommu_iova_to_phys(domain, dev_addr);
813
-	__dma_map_area(phys_to_virt(phys), size, dir);
814
-}
815
-
816
-static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,
817
-				   unsigned long offset, size_t size,
818
-				   enum dma_data_direction dir,
819
-				   unsigned long attrs)
820
-{
821
-	bool coherent = is_dma_coherent(dev, attrs);
822
-	int prot = dma_info_to_prot(dir, coherent, attrs);
823
-	dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);
824
-
825
-	if (!iommu_dma_mapping_error(dev, dev_addr) &&
826
-	    (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
827
-		__iommu_sync_single_for_device(dev, dev_addr, size, dir);
828
-
829
-	return dev_addr;
830
-}
831
-
832
-static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr,
833
-			       size_t size, enum dma_data_direction dir,
834
-			       unsigned long attrs)
835
-{
836
-	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
837
-		__iommu_sync_single_for_cpu(dev, dev_addr, size, dir);
838
-
839
-	iommu_dma_unmap_page(dev, dev_addr, size, dir, attrs);
840
-}
841
-
842
-static void __iommu_sync_sg_for_cpu(struct device *dev,
843
-				    struct scatterlist *sgl, int nelems,
844
-				    enum dma_data_direction dir)
845
-{
846
-	struct scatterlist *sg;
847
-	dma_addr_t iova = sg_dma_address(sgl);
848
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
849
-	int i;
850
-
851
-	if (!domain || iommu_is_iova_coherent(domain, iova))
852
-		return;
853
-	if (is_device_dma_coherent(dev))
854
-		return;
855
-	for_each_sg(sgl, sg, nelems, i)
856
-		__dma_unmap_area(sg_virt(sg), sg->length, dir);
857
-}
858
-
859
-static void __iommu_sync_sg_for_device(struct device *dev,
860
-				       struct scatterlist *sgl, int nelems,
861
-				       enum dma_data_direction dir)
862
-{
863
-	struct scatterlist *sg;
864
-	dma_addr_t iova = sg_dma_address(sgl);
865
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
866
-	int i;
867
-
868
-	if (!domain || iommu_is_iova_coherent(domain, iova))
869
-		return;
870
-	if (is_device_dma_coherent(dev))
871
-		return;
872
-	for_each_sg(sgl, sg, nelems, i)
873
-		__dma_map_area(sg_virt(sg), sg->length, dir);
874
-}
875
-
876
-static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
877
-				int nelems, enum dma_data_direction dir,
878
-				unsigned long attrs)
879
-{
880
-	bool coherent = is_device_dma_coherent(dev);
881
-
882
-	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
883
-		__iommu_sync_sg_for_device(dev, sgl, nelems, dir);
884
-
885
-	return iommu_dma_map_sg(dev, sgl, nelems,
886
-				dma_info_to_prot(dir, coherent, attrs));
887
-}
888
-
889
-static void __iommu_unmap_sg_attrs(struct device *dev,
890
-				   struct scatterlist *sgl, int nelems,
891
-				   enum dma_data_direction dir,
892
-				   unsigned long attrs)
893
-{
894
-	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
895
-		__iommu_sync_sg_for_cpu(dev, sgl, nelems, dir);
896
-
897
-	iommu_dma_unmap_sg(dev, sgl, nelems, dir, attrs);
898
-}
899
-
900
-static const struct dma_map_ops iommu_dma_ops = {
901
-	.alloc = __iommu_alloc_attrs,
902
-	.free = __iommu_free_attrs,
903
-	.mmap = __iommu_mmap_attrs,
904
-	.get_sgtable = __iommu_get_sgtable,
905
-	.map_page = __iommu_map_page,
906
-	.unmap_page = __iommu_unmap_page,
907
-	.map_sg = __iommu_map_sg_attrs,
908
-	.unmap_sg = __iommu_unmap_sg_attrs,
909
-	.sync_single_for_cpu = __iommu_sync_single_for_cpu,
910
-	.sync_single_for_device = __iommu_sync_single_for_device,
911
-	.sync_sg_for_cpu = __iommu_sync_sg_for_cpu,
912
-	.sync_sg_for_device = __iommu_sync_sg_for_device,
913
-	.map_resource = iommu_dma_map_resource,
914
-	.unmap_resource = iommu_dma_unmap_resource,
915
-	.mapping_error = iommu_dma_mapping_error,
916
-};
917
-
918
-static int __init __iommu_dma_init(void)
919
-{
920
-	return iommu_dma_init();
921
-}
922
-arch_initcall(__iommu_dma_init);
923
-
924
-static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
925
-				  const struct iommu_ops *ops)
926
-{
927
-	struct iommu_domain *domain;
928
-
929
-	if (!ops)
930
-		return;
931
-
932
-	/*
933
-	 * The IOMMU core code allocates the default DMA domain, which the
934
-	 * underlying IOMMU driver needs to support via the dma-iommu layer.
935
-	 */
936
-	domain = iommu_get_domain_for_dev(dev);
937
-
938
-	if (!domain)
939
-		goto out_err;
940
-
941
-	if (domain->type == IOMMU_DOMAIN_DMA) {
942
-		if (iommu_dma_init_domain(domain, dma_base, size, dev))
943
-			goto out_err;
944
-
945
-		dev->dma_ops = &iommu_dma_ops;
946
-	}
947
-
948
-	return;
949
-
950
-out_err:
951
-	 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
952
-		 dev_name(dev));
953
-}
954
-
95536 void arch_teardown_dma_ops(struct device *dev)
95637 {
95738 	dev->dma_ops = NULL;
95839 }
959
-
960
-#else
961
-
962
-static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
963
-				  const struct iommu_ops *iommu)
964
-{ }
965
-
966
-#endif  /* CONFIG_IOMMU_DMA */
967
-
968
-static void arm_iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size);
40
+#endif
96941 
97042 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
97143 			const struct iommu_ops *iommu, bool coherent)
97244 {
973
-	if (!dev->dma_ops) {
974
-		if (dev->removed_mem)
975
-			set_dma_ops(dev, &removed_dma_ops);
976
-		else
977
-			dev->dma_ops = &arm64_swiotlb_dma_ops;
45
+	int cls = cache_line_size_of_cpu();
46
+
47
+	WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN,
48
+		   TAINT_CPU_OUT_OF_SPEC,
49
+		   "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)",
50
+		   dev_driver_string(dev), dev_name(dev),
51
+		   ARCH_DMA_MINALIGN, cls);
52
+
53
+	dev->dma_coherent = coherent;
54
+	if (iommu) {
55
+		iommu_setup_dma_ops(dev, dma_base, size);
56
+		trace_android_vh_iommu_setup_dma_ops(dev, dma_base, size);
57
+		trace_android_rvh_iommu_setup_dma_ops(dev, dma_base, size);
97858 	}
97959 
980
-	dev->archdata.dma_coherent = coherent;
981
-	if (of_parse_phandle(dev->of_node, "qcom,iommu-group", 0))
982
-		arm_iommu_setup_dma_ops(dev, dma_base, size);
983
-	else
984
-		__iommu_setup_dma_ops(dev, dma_base, size, iommu);
60
+	/* Allow vendor modules to opt-in for the 2454944 erratum workaround */
61
+	trace_android_rvh_setup_dma_ops(dev);
98562 
98663 #ifdef CONFIG_XEN
987
-	if (xen_initial_domain()) {
988
-		dev->archdata.dev_dma_ops = dev->dma_ops;
989
-		dev->dma_ops = xen_dma_ops;
990
-	}
64
+	if (xen_initial_domain())
65
+		dev->dma_ops = &xen_swiotlb_dma_ops;
99166 #endif
99267 }
993
-EXPORT_SYMBOL_GPL(arch_setup_dma_ops);
99468 
995
-#ifdef CONFIG_ARM64_DMA_USE_IOMMU
996
-
997
-/* guards initialization of default_domain->iova_cookie */
998
-static DEFINE_MUTEX(iommu_dma_init_mutex);
999
-
1000
-static int
1001
-iommu_init_mapping(struct device *dev, struct dma_iommu_mapping *mapping)
1002
-{
1003
-	struct iommu_domain *domain = mapping->domain;
1004
-	dma_addr_t dma_base = mapping->base;
1005
-	u64 size = mapping->bits << PAGE_SHIFT;
1006
-	int ret;
1007
-	bool own_cookie;
1008
-
1009
-	/*
1010
-	 * if own_cookie is false, then we are sharing the iova_cookie with
1011
-	 * another driver, and should not free it on error. Cleanup will be
1012
-	 * done when the iommu_domain is freed.
1013
-	 */
1014
-	own_cookie = !domain->iova_cookie;
1015
-
1016
-	if (own_cookie) {
1017
-		ret = iommu_get_dma_cookie(domain);
1018
-		if (ret) {
1019
-			dev_err(dev, "iommu_get_dma_cookie failed: %d\n", ret);
1020
-			return ret;
1021
-		}
1022
-	}
1023
-
1024
-	ret = iommu_dma_init_domain(domain, dma_base, size, dev);
1025
-	if (ret) {
1026
-		dev_err(dev, "iommu_dma_init_domain failed: %d\n", ret);
1027
-		if (own_cookie)
1028
-			iommu_put_dma_cookie(domain);
1029
-		return ret;
1030
-	}
1031
-
1032
-	mapping->ops = &iommu_dma_ops;
1033
-	return 0;
1034
-}
1035
-
1036
-static int arm_iommu_get_dma_cookie(struct device *dev,
1037
-				    struct dma_iommu_mapping *mapping)
1038
-{
1039
-	int s1_bypass = 0, is_fast = 0;
1040
-	int err = 0;
1041
-
1042
-	mutex_lock(&iommu_dma_init_mutex);
1043
-
1044
-	iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_S1_BYPASS,
1045
-					&s1_bypass);
1046
-	iommu_domain_get_attr(mapping->domain, DOMAIN_ATTR_FAST, &is_fast);
1047
-
1048
-	if (s1_bypass)
1049
-		mapping->ops = &arm64_swiotlb_dma_ops;
1050
-	else if (is_fast)
1051
-		err = fast_smmu_init_mapping(dev, mapping);
1052
-	else
1053
-		err = iommu_init_mapping(dev, mapping);
1054
-
1055
-	mutex_unlock(&iommu_dma_init_mutex);
1056
-	return err;
1057
-}
1058
-
1059
-/*
1060
- * Checks for "qcom,iommu-dma-addr-pool" property.
1061
- * If not present, leaves dma_addr and dma_size unmodified.
1062
- */
1063
-static void arm_iommu_get_dma_window(struct device *dev, u64 *dma_addr,
1064
-					u64 *dma_size)
1065
-{
1066
-	struct device_node *np;
1067
-	int naddr, nsize, len;
1068
-	const __be32 *ranges;
1069
-
1070
-	if (!dev->of_node)
1071
-		return;
1072
-
1073
-	np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0);
1074
-	if (!np)
1075
-		np = dev->of_node;
1076
-
1077
-	ranges = of_get_property(np, "qcom,iommu-dma-addr-pool", &len);
1078
-	if (!ranges)
1079
-		return;
1080
-
1081
-	len /= sizeof(u32);
1082
-	naddr = of_n_addr_cells(np);
1083
-	nsize = of_n_size_cells(np);
1084
-	if (len < naddr + nsize) {
1085
-		dev_err(dev, "Invalid length for qcom,iommu-dma-addr-pool, expected %d cells\n",
1086
-			naddr + nsize);
1087
-		return;
1088
-	}
1089
-	if (naddr == 0 || nsize == 0) {
1090
-		dev_err(dev, "Invalid #address-cells %d or #size-cells %d\n",
1091
-			naddr, nsize);
1092
-		return;
1093
-	}
1094
-
1095
-	*dma_addr = of_read_number(ranges, naddr);
1096
-	*dma_size = of_read_number(ranges + naddr, nsize);
1097
-}
1098
-
1099
-static void arm_iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)
1100
-{
1101
-	struct iommu_domain *domain;
1102
-	struct iommu_group *group;
1103
-	struct dma_iommu_mapping mapping = {0};
1104
-
1105
-	group = dev->iommu_group;
1106
-	if (!group)
1107
-		return;
1108
-
1109
-	arm_iommu_get_dma_window(dev, &dma_base, &size);
1110
-
1111
-	domain = iommu_get_domain_for_dev(dev);
1112
-	if (!domain)
1113
-		return;
1114
-
1115
-	/* Allow iommu-debug to call arch_setup_dma_ops to reconfigure itself */
1116
-	if (domain->type != IOMMU_DOMAIN_DMA &&
1117
-	    !of_device_is_compatible(dev->of_node, "iommu-debug-test")) {
1118
-		dev_err(dev, "Invalid iommu domain type!\n");
1119
-		return;
1120
-	}
1121
-
1122
-	mapping.base = dma_base;
1123
-	mapping.bits = size >> PAGE_SHIFT;
1124
-	mapping.domain = domain;
1125
-
1126
-	if (arm_iommu_get_dma_cookie(dev, &mapping)) {
1127
-		dev_err(dev, "Failed to get dma cookie\n");
1128
-		return;
1129
-	}
1130
-
1131
-	set_dma_ops(dev, mapping.ops);
1132
-}
1133
-
1134
-#else /*!CONFIG_ARM64_DMA_USE_IOMMU */
1135
-
1136
-static void arm_iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)
1137
-{
1138
-}
69
+#ifdef CONFIG_NO_GKI
70
+EXPORT_SYMBOL(__dma_map_area);
71
+EXPORT_SYMBOL(__dma_unmap_area);
113972 #endif