mk-rootfs.sh

hc

2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376

 kernel/kernel/dma/direct.c
..
..
@@ -1,151 +1,395 @@
1
1
 // SPDX-License-Identifier: GPL-2.0
2
2
 /*
3
- * DMA operations that map physical memory directly without using an IOMMU or
4
- * flushing caches.
3
+ * Copyright (C) 2018-2020 Christoph Hellwig.
4
+ *
5
+ * DMA operations that map physical memory directly without using an IOMMU.
5
6
  */
7
+#include <linux/memblock.h> /* for max_pfn */
6
8
 #include <linux/export.h>
7
9
 #include <linux/mm.h>
8
-#include <linux/dma-direct.h>
10
+#include <linux/dma-map-ops.h>
9
11
 #include <linux/scatterlist.h>
10
-#include <linux/dma-contiguous.h>
11
12
 #include <linux/pfn.h>
13
+#include <linux/vmalloc.h>
12
14
 #include <linux/set_memory.h>
13
-
14
-#define DIRECT_MAPPING_ERROR		0
15
+#include <linux/slab.h>
16
+#include "direct.h"
15
17
 
16
18
 /*
17
- * Most architectures use ZONE_DMA for the first 16 Megabytes, but
18
- * some use it for entirely different regions:
19
+ * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use
20
+ * it for entirely different regions. In that case the arch code needs to
21
+ * override the variable below for dma-direct to work properly.
19
22
  */
20
-#ifndef ARCH_ZONE_DMA_BITS
21
-#define ARCH_ZONE_DMA_BITS 24
22
-#endif
23
+unsigned int zone_dma_bits __ro_after_init = 24;
23
24
 
24
-/*
25
- * For AMD SEV all DMA must be to unencrypted addresses.
26
- */
27
-static inline bool force_dma_unencrypted(void)
25
+static inline dma_addr_t phys_to_dma_direct(struct device *dev,
26
+		phys_addr_t phys)
28
27
 {
29
-	return sev_active();
28
+	if (force_dma_unencrypted(dev))
29
+		return phys_to_dma_unencrypted(dev, phys);
30
+	return phys_to_dma(dev, phys);
30
31
 }
31
32
 
32
-static bool
33
-check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
34
-		const char *caller)
33
+static inline struct page *dma_direct_to_page(struct device *dev,
34
+		dma_addr_t dma_addr)
35
35
 {
36
-	if (unlikely(dev && !dma_capable(dev, dma_addr, size))) {
37
-		if (!dev->dma_mask) {
38
-			dev_err(dev,
39
-				"%s: call on device without dma_mask\n",
40
-				caller);
41
-			return false;
42
-		}
36
+	return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr)));
37
+}
43
38
 
44
-		if (*dev->dma_mask >= DMA_BIT_MASK(32)) {
45
-			dev_err(dev,
46
-				"%s: overflow %pad+%zu of device mask %llx\n",
47
-				caller, &dma_addr, size, *dev->dma_mask);
48
-		}
49
-		return false;
50
-	}
51
-	return true;
39
+u64 dma_direct_get_required_mask(struct device *dev)
40
+{
41
+	phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT;
42
+	u64 max_dma = phys_to_dma_direct(dev, phys);
43
+
44
+	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
45
+}
46
+
47
+static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
48
+				  u64 *phys_limit)
49
+{
50
+	u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
51
+
52
+	/*
53
+	 * Optimistically try the zone that the physical address mask falls
54
+	 * into first.  If that returns memory that isn't actually addressable
55
+	 * we will fallback to the next lower zone and try again.
56
+	 *
57
+	 * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding
58
+	 * zones.
59
+	 */
60
+	*phys_limit = dma_to_phys(dev, dma_limit);
61
+	if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits))
62
+		return GFP_DMA;
63
+	if (*phys_limit <= DMA_BIT_MASK(32) &&
64
+		!zone_dma32_are_empty())
65
+		return GFP_DMA32;
66
+	return 0;
52
67
 }
53
68
 
54
69
 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
55
70
 {
56
-	dma_addr_t addr = force_dma_unencrypted() ?
57
-		__phys_to_dma(dev, phys) : phys_to_dma(dev, phys);
58
-	return addr + size - 1 <= dev->coherent_dma_mask;
71
+	dma_addr_t dma_addr = phys_to_dma_direct(dev, phys);
72
+
73
+	if (dma_addr == DMA_MAPPING_ERROR)
74
+		return false;
75
+	return dma_addr + size - 1 <=
76
+		min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
59
77
 }
60
78
 
61
-void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
62
-		gfp_t gfp, unsigned long attrs)
79
+static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
80
+		gfp_t gfp)
63
81
 {
64
-	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
65
-	int page_order = get_order(size);
82
+	int node = dev_to_node(dev);
66
83
 	struct page *page = NULL;
67
-	void *ret;
84
+	u64 phys_limit;
68
85
 
69
-	/* we always manually zero the memory once we are done: */
70
-	gfp &= ~__GFP_ZERO;
86
+	WARN_ON_ONCE(!PAGE_ALIGNED(size));
71
87
 
72
-	/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */
73
-	if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
74
-		gfp |= GFP_DMA;
75
-	if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
76
-		gfp |= GFP_DMA32;
77
-
78
-again:
79
-	/* CMA can be used only in the context which permits sleeping */
80
-	if (gfpflags_allow_blocking(gfp)) {
81
-		page = dma_alloc_from_contiguous(dev, count, page_order,
82
-						 gfp & __GFP_NOWARN);
83
-		if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
84
-			dma_release_from_contiguous(dev, page, count);
85
-			page = NULL;
86
-		}
87
-	}
88
-	if (!page)
89
-		page = alloc_pages_node(dev_to_node(dev), gfp, page_order);
90
-
88
+	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
89
+					   &phys_limit);
90
+	page = dma_alloc_contiguous(dev, size, gfp);
91
91
 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
92
-		__free_pages(page, page_order);
92
+		dma_free_contiguous(dev, page, size);
93
+		page = NULL;
94
+	}
95
+again:
96
+	if (!page)
97
+		page = alloc_pages_node(node, gfp, get_order(size));
98
+	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
99
+		dma_free_contiguous(dev, page, size);
93
100
 		page = NULL;
94
101
 
95
102
 		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
96
-		    dev->coherent_dma_mask < DMA_BIT_MASK(64) &&
97
-		    !(gfp & (GFP_DMA32 | GFP_DMA))) {
103
+		    phys_limit < DMA_BIT_MASK(64) &&
104
+		    !(gfp & (GFP_DMA32 | GFP_DMA)) &&
105
+		    !zone_dma32_are_empty()) {
98
106
 			gfp |= GFP_DMA32;
99
107
 			goto again;
100
108
 		}
101
109
 
102
-		if (IS_ENABLED(CONFIG_ZONE_DMA) &&
103
-		    dev->coherent_dma_mask < DMA_BIT_MASK(32) &&
104
-		    !(gfp & GFP_DMA)) {
110
+		if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) {
105
111
 			gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
106
112
 			goto again;
107
113
 		}
108
114
 	}
109
115
 
116
+	return page;
117
+}
118
+
119
+static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
120
+		dma_addr_t *dma_handle, gfp_t gfp)
121
+{
122
+	struct page *page;
123
+	u64 phys_mask;
124
+	void *ret;
125
+
126
+	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
127
+					   &phys_mask);
128
+	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
110
129
 	if (!page)
111
130
 		return NULL;
112
-	ret = page_address(page);
113
-	if (force_dma_unencrypted()) {
114
-		set_memory_decrypted((unsigned long)ret, 1 << page_order);
115
-		*dma_handle = __phys_to_dma(dev, page_to_phys(page));
116
-	} else {
117
-		*dma_handle = phys_to_dma(dev, page_to_phys(page));
118
-	}
119
-	memset(ret, 0, size);
131
+	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
120
132
 	return ret;
121
133
 }
122
134
 
123
-/*
124
- * NOTE: this function must never look at the dma_addr argument, because we want
125
- * to be able to use it as a helper for iommu implementations as well.
126
- */
127
-void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
128
-		dma_addr_t dma_addr, unsigned long attrs)
135
+void *dma_direct_alloc(struct device *dev, size_t size,
136
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
129
137
 {
130
-	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
131
-	unsigned int page_order = get_order(size);
138
+	struct page *page;
139
+	void *ret;
140
+	int err;
132
141
 
133
-	if (force_dma_unencrypted())
134
-		set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
135
-	if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
136
-		free_pages((unsigned long)cpu_addr, page_order);
142
+	size = PAGE_ALIGN(size);
143
+	if (attrs & DMA_ATTR_NO_WARN)
144
+		gfp |= __GFP_NOWARN;
145
+
146
+	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
147
+	    !force_dma_unencrypted(dev)) {
148
+		page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);
149
+		if (!page)
150
+			return NULL;
151
+		/* remove any dirty cache lines on the kernel alias */
152
+		if (!PageHighMem(page))
153
+			arch_dma_prep_coherent(page, size);
154
+		*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
155
+		/* return the page pointer as the opaque cookie */
156
+		return page;
157
+	}
158
+
159
+	if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
160
+	    !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
161
+	    !dev_is_dma_coherent(dev))
162
+		return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
163
+
164
+	/*
165
+	 * Remapping or decrypting memory may block. If either is required and
166
+	 * we can't block, allocate the memory from the atomic pools.
167
+	 */
168
+	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
169
+	    !gfpflags_allow_blocking(gfp) &&
170
+	    (force_dma_unencrypted(dev) ||
171
+	     (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && !dev_is_dma_coherent(dev))))
172
+		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
173
+
174
+	/* we always manually zero the memory once we are done */
175
+	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);
176
+	if (!page)
177
+		return NULL;
178
+
179
+	if ((IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
180
+	     !dev_is_dma_coherent(dev)) ||
181
+	    (IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) {
182
+		/* remove any dirty cache lines on the kernel alias */
183
+		arch_dma_prep_coherent(page, size);
184
+
185
+		/* create a coherent mapping */
186
+		ret = dma_common_contiguous_remap(page, size,
187
+				dma_pgprot(dev, PAGE_KERNEL, attrs),
188
+				__builtin_return_address(0));
189
+		if (!ret)
190
+			goto out_free_pages;
191
+		if (force_dma_unencrypted(dev)) {
192
+			err = set_memory_decrypted((unsigned long)ret,
193
+						   PFN_UP(size));
194
+			if (err)
195
+				goto out_free_pages;
196
+		}
197
+		memset(ret, 0, size);
198
+		goto done;
199
+	}
200
+
201
+	if (PageHighMem(page)) {
202
+		/*
203
+		 * Depending on the cma= arguments and per-arch setup
204
+		 * dma_alloc_contiguous could return highmem pages.
205
+		 * Without remapping there is no way to return them here,
206
+		 * so log an error and fail.
207
+		 */
208
+		dev_info(dev, "Rejecting highmem page from CMA.\n");
209
+		goto out_free_pages;
210
+	}
211
+
212
+	ret = page_address(page);
213
+	if (force_dma_unencrypted(dev)) {
214
+		err = set_memory_decrypted((unsigned long)ret,
215
+					   PFN_UP(size));
216
+		if (err)
217
+			goto out_free_pages;
218
+	}
219
+
220
+	memset(ret, 0, size);
221
+
222
+	if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
223
+	    !dev_is_dma_coherent(dev)) {
224
+		arch_dma_prep_coherent(page, size);
225
+		ret = arch_dma_set_uncached(ret, size);
226
+		if (IS_ERR(ret))
227
+			goto out_encrypt_pages;
228
+	}
229
+done:
230
+	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
231
+	return ret;
232
+
233
+out_encrypt_pages:
234
+	if (force_dma_unencrypted(dev)) {
235
+		err = set_memory_encrypted((unsigned long)page_address(page),
236
+					   PFN_UP(size));
237
+		/* If memory cannot be re-encrypted, it must be leaked */
238
+		if (err)
239
+			return NULL;
240
+	}
241
+out_free_pages:
242
+	dma_free_contiguous(dev, page, size);
243
+	return NULL;
137
244
 }
138
245
 
139
-dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
140
-		unsigned long offset, size_t size, enum dma_data_direction dir,
141
-		unsigned long attrs)
246
+void dma_direct_free(struct device *dev, size_t size,
247
+		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
142
248
 {
143
-	dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset;
249
+	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
250
+	    !force_dma_unencrypted(dev)) {
251
+		/* cpu_addr is a struct page cookie, not a kernel address */
252
+		dma_free_contiguous(dev, cpu_addr, size);
253
+		return;
254
+	}
144
255
 
145
-	if (!check_addr(dev, dma_addr, size, __func__))
146
-		return DIRECT_MAPPING_ERROR;
147
-	return dma_addr;
256
+	if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
257
+	    !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
258
+	    !dev_is_dma_coherent(dev)) {
259
+		arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
260
+		return;
261
+	}
262
+
263
+	/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
264
+	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
265
+	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
266
+		return;
267
+
268
+	if (force_dma_unencrypted(dev))
269
+		set_memory_encrypted((unsigned long)cpu_addr, PFN_UP(size));
270
+
271
+	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr))
272
+		vunmap(cpu_addr);
273
+	else if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
274
+		arch_dma_clear_uncached(cpu_addr, size);
275
+
276
+	dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
148
277
 }
278
+
279
+struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
280
+		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
281
+{
282
+	struct page *page;
283
+	void *ret;
284
+
285
+	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
286
+	    force_dma_unencrypted(dev) && !gfpflags_allow_blocking(gfp))
287
+		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
288
+
289
+	page = __dma_direct_alloc_pages(dev, size, gfp);
290
+	if (!page)
291
+		return NULL;
292
+	if (PageHighMem(page)) {
293
+		/*
294
+		 * Depending on the cma= arguments and per-arch setup
295
+		 * dma_alloc_contiguous could return highmem pages.
296
+		 * Without remapping there is no way to return them here,
297
+		 * so log an error and fail.
298
+		 */
299
+		dev_info(dev, "Rejecting highmem page from CMA.\n");
300
+		goto out_free_pages;
301
+	}
302
+
303
+	ret = page_address(page);
304
+	if (force_dma_unencrypted(dev)) {
305
+		if (set_memory_decrypted((unsigned long)ret, PFN_UP(size)))
306
+			goto out_free_pages;
307
+	}
308
+	memset(ret, 0, size);
309
+	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
310
+	return page;
311
+out_free_pages:
312
+	dma_free_contiguous(dev, page, size);
313
+	return NULL;
314
+}
315
+
316
+void dma_direct_free_pages(struct device *dev, size_t size,
317
+		struct page *page, dma_addr_t dma_addr,
318
+		enum dma_data_direction dir)
319
+{
320
+	void *vaddr = page_address(page);
321
+
322
+	/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
323
+	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
324
+	    dma_free_from_pool(dev, vaddr, size))
325
+		return;
326
+
327
+	if (force_dma_unencrypted(dev))
328
+		set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));
329
+
330
+	dma_free_contiguous(dev, page, size);
331
+}
332
+
333
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
334
+    defined(CONFIG_SWIOTLB)
335
+void dma_direct_sync_sg_for_device(struct device *dev,
336
+		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
337
+{
338
+	struct scatterlist *sg;
339
+	int i;
340
+
341
+	for_each_sg(sgl, sg, nents, i) {
342
+		phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
343
+
344
+		if (unlikely(is_swiotlb_buffer(paddr)))
345
+			swiotlb_tbl_sync_single(dev, paddr, sg->length,
346
+					dir, SYNC_FOR_DEVICE);
347
+
348
+		if (!dev_is_dma_coherent(dev))
349
+			arch_sync_dma_for_device(paddr, sg->length,
350
+					dir);
351
+	}
352
+}
353
+#endif
354
+
355
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
356
+    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
357
+    defined(CONFIG_SWIOTLB)
358
+void dma_direct_sync_sg_for_cpu(struct device *dev,
359
+		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
360
+{
361
+	struct scatterlist *sg;
362
+	int i;
363
+
364
+	for_each_sg(sgl, sg, nents, i) {
365
+		phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
366
+
367
+		if (!dev_is_dma_coherent(dev))
368
+			arch_sync_dma_for_cpu(paddr, sg->length, dir);
369
+
370
+		if (unlikely(is_swiotlb_buffer(paddr)))
371
+			swiotlb_tbl_sync_single(dev, paddr, sg->length, dir,
372
+					SYNC_FOR_CPU);
373
+
374
+		if (dir == DMA_FROM_DEVICE)
375
+			arch_dma_mark_clean(paddr, sg->length);
376
+	}
377
+
378
+	if (!dev_is_dma_coherent(dev))
379
+		arch_sync_dma_for_cpu_all();
380
+}
381
+
382
+void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
383
+		int nents, enum dma_data_direction dir, unsigned long attrs)
384
+{
385
+	struct scatterlist *sg;
386
+	int i;
387
+
388
+	for_each_sg(sgl, sg, nents, i)
389
+		dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
390
+			     attrs);
391
+}
392
+#endif
149
393
 
150
394
 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
151
395
 		enum dma_data_direction dir, unsigned long attrs)
..
..
@@ -154,61 +398,151 @@
154
398
 	struct scatterlist *sg;
155
399
 
156
400
 	for_each_sg(sgl, sg, nents, i) {
157
-		BUG_ON(!sg_page(sg));
158
-
159
-		sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg));
160
-		if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__))
161
-			return 0;
401
+		sg->dma_address = dma_direct_map_page(dev, sg_page(sg),
402
+				sg->offset, sg->length, dir, attrs);
403
+		if (sg->dma_address == DMA_MAPPING_ERROR)
404
+			goto out_unmap;
162
405
 		sg_dma_len(sg) = sg->length;
163
406
 	}
164
407
 
165
408
 	return nents;
409
+
410
+out_unmap:
411
+	dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
412
+	return 0;
413
+}
414
+
415
+dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
416
+		size_t size, enum dma_data_direction dir, unsigned long attrs)
417
+{
418
+	dma_addr_t dma_addr = paddr;
419
+
420
+	if (unlikely(!dma_capable(dev, dma_addr, size, false))) {
421
+		dev_err_once(dev,
422
+			     "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
423
+			     &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
424
+		WARN_ON_ONCE(1);
425
+		return DMA_MAPPING_ERROR;
426
+	}
427
+
428
+	return dma_addr;
429
+}
430
+
431
+int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
432
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
433
+		unsigned long attrs)
434
+{
435
+	struct page *page = dma_direct_to_page(dev, dma_addr);
436
+	int ret;
437
+
438
+	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
439
+	if (!ret)
440
+		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
441
+	return ret;
442
+}
443
+
444
+bool dma_direct_can_mmap(struct device *dev)
445
+{
446
+	return dev_is_dma_coherent(dev) ||
447
+		IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP);
448
+}
449
+
450
+int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
451
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
452
+		unsigned long attrs)
453
+{
454
+	unsigned long user_count = vma_pages(vma);
455
+	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
456
+	unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
457
+	int ret = -ENXIO;
458
+
459
+	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
460
+
461
+	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
462
+		return ret;
463
+
464
+	if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)
465
+		return -ENXIO;
466
+	return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
467
+			user_count << PAGE_SHIFT, vma->vm_page_prot);
166
468
 }
167
469
 
168
470
 int dma_direct_supported(struct device *dev, u64 mask)
169
471
 {
170
-#ifdef CONFIG_ZONE_DMA
171
-	/*
172
-	 * This check needs to be against the actual bit mask value, so
173
-	 * use __phys_to_dma() here so that the SME encryption mask isn't
174
-	 * part of the check.
175
-	 */
176
-	if (mask < __phys_to_dma(dev, DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)))
177
-		return 0;
178
-#else
472
+	u64 min_mask = (max_pfn - 1) << PAGE_SHIFT;
473
+
179
474
 	/*
180
475
 	 * Because 32-bit DMA masks are so common we expect every architecture
181
476
 	 * to be able to satisfy them - either by not supporting more physical
182
477
 	 * memory, or by providing a ZONE_DMA32.  If neither is the case, the
183
478
 	 * architecture needs to use an IOMMU instead of the direct mapping.
184
-	 *
185
-	 * This check needs to be against the actual bit mask value, so
186
-	 * use __phys_to_dma() here so that the SME encryption mask isn't
479
+	 */
480
+	if (mask >= DMA_BIT_MASK(32))
481
+		return 1;
482
+
483
+	/*
484
+	 * This check needs to be against the actual bit mask value, so use
485
+	 * phys_to_dma_unencrypted() here so that the SME encryption mask isn't
187
486
 	 * part of the check.
188
487
 	 */
189
-	if (mask < __phys_to_dma(dev, DMA_BIT_MASK(32)))
190
-		return 0;
191
-#endif
192
-	/*
193
-	 * Upstream PCI/PCIe bridges or SoC interconnects may not carry
194
-	 * as many DMA address bits as the device itself supports.
195
-	 */
196
-	if (dev->bus_dma_mask && mask > dev->bus_dma_mask)
197
-		return 0;
198
-	return 1;
488
+	if (IS_ENABLED(CONFIG_ZONE_DMA))
489
+		min_mask = min_t(u64, min_mask, DMA_BIT_MASK(zone_dma_bits));
490
+	return mask >= phys_to_dma_unencrypted(dev, min_mask);
199
491
 }
200
492
 
201
-int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
493
+size_t dma_direct_max_mapping_size(struct device *dev)
202
494
 {
203
-	return dma_addr == DIRECT_MAPPING_ERROR;
495
+	/* If SWIOTLB is active, use its maximum mapping size */
496
+	if (is_swiotlb_active() &&
497
+	    (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
498
+		return swiotlb_max_mapping_size(dev);
499
+	return SIZE_MAX;
204
500
 }
205
501
 
206
-const struct dma_map_ops dma_direct_ops = {
207
-	.alloc			= dma_direct_alloc,
208
-	.free			= dma_direct_free,
209
-	.map_page		= dma_direct_map_page,
210
-	.map_sg			= dma_direct_map_sg,
211
-	.dma_supported		= dma_direct_supported,
212
-	.mapping_error		= dma_direct_mapping_error,
213
-};
214
-EXPORT_SYMBOL(dma_direct_ops);
502
+bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr)
503
+{
504
+	return !dev_is_dma_coherent(dev) ||
505
+		is_swiotlb_buffer(dma_to_phys(dev, dma_addr));
506
+}
507
+
508
+/**
509
+ * dma_direct_set_offset - Assign scalar offset for a single DMA range.
510
+ * @dev:	device pointer; needed to "own" the alloced memory.
511
+ * @cpu_start:  beginning of memory region covered by this offset.
512
+ * @dma_start:  beginning of DMA/PCI region covered by this offset.
513
+ * @size:	size of the region.
514
+ *
515
+ * This is for the simple case of a uniform offset which cannot
516
+ * be discovered by "dma-ranges".
517
+ *
518
+ * It returns -ENOMEM if out of memory, -EINVAL if a map
519
+ * already exists, 0 otherwise.
520
+ *
521
+ * Note: any call to this from a driver is a bug.  The mapping needs
522
+ * to be described by the device tree or other firmware interfaces.
523
+ */
524
+int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
525
+			 dma_addr_t dma_start, u64 size)
526
+{
527
+	struct bus_dma_region *map;
528
+	u64 offset = (u64)cpu_start - (u64)dma_start;
529
+
530
+	if (dev->dma_range_map) {
531
+		dev_err(dev, "attempt to add DMA range to existing map\n");
532
+		return -EINVAL;
533
+	}
534
+
535
+	if (!offset)
536
+		return 0;
537
+
538
+	map = kcalloc(2, sizeof(*map), GFP_KERNEL);
539
+	if (!map)
540
+		return -ENOMEM;
541
+	map[0].cpu_start = cpu_start;
542
+	map[0].dma_start = dma_start;
543
+	map[0].offset = offset;
544
+	map[0].size = size;
545
+	dev->dma_range_map = map;
546
+	return 0;
547
+}
548
+EXPORT_SYMBOL_GPL(dma_direct_set_offset);