forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/kernel/dma/contiguous.c
....@@ -5,6 +5,34 @@
55  * Written by:
66  *	Marek Szyprowski <m.szyprowski@samsung.com>
77  *	Michal Nazarewicz <mina86@mina86.com>
8
+ *
9
+ * Contiguous Memory Allocator
10
+ *
11
+ *   The Contiguous Memory Allocator (CMA) makes it possible to
12
+ *   allocate big contiguous chunks of memory after the system has
13
+ *   booted.
14
+ *
15
+ * Why is it needed?
16
+ *
17
+ *   Various devices on embedded systems have no scatter-getter and/or
18
+ *   IO map support and require contiguous blocks of memory to
19
+ *   operate.  They include devices such as cameras, hardware video
20
+ *   coders, etc.
21
+ *
22
+ *   Such devices often require big memory buffers (a full HD frame
23
+ *   is, for instance, more then 2 mega pixels large, i.e. more than 6
24
+ *   MB of memory), which makes mechanisms such as kmalloc() or
25
+ *   alloc_page() ineffective.
26
+ *
27
+ *   At the same time, a solution where a big memory region is
28
+ *   reserved for a device is suboptimal since often more memory is
29
+ *   reserved then strictly required and, moreover, the memory is
30
+ *   inaccessible to page system even if device drivers don't use it.
31
+ *
32
+ *   CMA tries to solve this issue by operating on memory regions
33
+ *   where only movable pages can be allocated from.  This way, kernel
34
+ *   can use the memory for pagecache and when device driver requests
35
+ *   it, allocated pages can be migrated.
836  */
937 
1038 #define pr_fmt(fmt) "cma: " fmt
....@@ -16,13 +44,13 @@
1644 #endif
1745 
1846 #include <asm/page.h>
19
-#include <asm/dma-contiguous.h>
2047 
2148 #include <linux/memblock.h>
2249 #include <linux/err.h>
2350 #include <linux/sizes.h>
24
-#include <linux/dma-contiguous.h>
51
+#include <linux/dma-map-ops.h>
2552 #include <linux/cma.h>
53
+#include <trace/hooks/mm.h>
2654 
2755 #ifdef CONFIG_CMA_SIZE_MBYTES
2856 #define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES
....@@ -31,7 +59,7 @@
3159 #endif
3260 
3361 struct cma *dma_contiguous_default_area;
34
-EXPORT_SYMBOL(dma_contiguous_default_area);
62
+EXPORT_SYMBOL_GPL(dma_contiguous_default_area);
3563 
3664 /*
3765  * Default global CMA area size can be defined in kernel's .config.
....@@ -43,10 +71,11 @@
4371  * Users, who want to set the size of global CMA area for their system
4472  * should use cma= kernel parameter.
4573  */
46
-static const phys_addr_t size_bytes = (phys_addr_t)CMA_SIZE_MBYTES * SZ_1M;
47
-static phys_addr_t size_cmdline = -1;
48
-static phys_addr_t base_cmdline;
49
-static phys_addr_t limit_cmdline;
74
+static const phys_addr_t size_bytes __initconst =
75
+	(phys_addr_t)CMA_SIZE_MBYTES * SZ_1M;
76
+static phys_addr_t  size_cmdline __initdata = -1;
77
+static phys_addr_t base_cmdline __initdata;
78
+static phys_addr_t limit_cmdline __initdata;
5079 
5180 static int __init early_cma(char *p)
5281 {
....@@ -69,20 +98,24 @@
6998 }
7099 early_param("cma", early_cma);
71100 
101
+#ifdef CONFIG_DMA_PERNUMA_CMA
102
+
103
+static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES];
104
+static phys_addr_t pernuma_size_bytes __initdata;
105
+
106
+static int __init early_cma_pernuma(char *p)
107
+{
108
+	pernuma_size_bytes = memparse(p, &p);
109
+	return 0;
110
+}
111
+early_param("cma_pernuma", early_cma_pernuma);
112
+#endif
113
+
72114 #ifdef CONFIG_CMA_SIZE_PERCENTAGE
73115 
74116 static phys_addr_t __init __maybe_unused cma_early_percent_memory(void)
75117 {
76
-	struct memblock_region *reg;
77
-	unsigned long total_pages = 0;
78
-
79
-	/*
80
-	 * We cannot use memblock_phys_mem_size() here, because
81
-	 * memblock_analyze() has not been called yet.
82
-	 */
83
-	for_each_memblock(memory, reg)
84
-		total_pages += memblock_region_memory_end_pfn(reg) -
85
-			       memblock_region_memory_base_pfn(reg);
118
+	unsigned long total_pages = PHYS_PFN(memblock_phys_mem_size());
86119 
87120 	return (total_pages * CONFIG_CMA_SIZE_PERCENTAGE / 100) << PAGE_SHIFT;
88121 }
....@@ -94,6 +127,34 @@
94127 	return 0;
95128 }
96129 
130
+#endif
131
+
132
+#ifdef CONFIG_DMA_PERNUMA_CMA
133
+void __init dma_pernuma_cma_reserve(void)
134
+{
135
+	int nid;
136
+
137
+	if (!pernuma_size_bytes)
138
+		return;
139
+
140
+	for_each_online_node(nid) {
141
+		int ret;
142
+		char name[CMA_MAX_NAME];
143
+		struct cma **cma = &dma_contiguous_pernuma_area[nid];
144
+
145
+		snprintf(name, sizeof(name), "pernuma%d", nid);
146
+		ret = cma_declare_contiguous_nid(0, pernuma_size_bytes, 0, 0,
147
+						 0, false, name, cma, nid);
148
+		if (ret) {
149
+			pr_warn("%s: reservation failed: err %d, node %d", __func__,
150
+				ret, nid);
151
+			continue;
152
+		}
153
+
154
+		pr_debug("%s: reserved %llu MiB on node %d\n", __func__,
155
+			(unsigned long long)pernuma_size_bytes / SZ_1M, nid);
156
+	}
157
+}
97158 #endif
98159 
99160 /**
....@@ -141,6 +202,11 @@
141202 					    &dma_contiguous_default_area,
142203 					    fixed);
143204 	}
205
+}
206
+
207
+void __weak
208
+dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
209
+{
144210 }
145211 
146212 /**
....@@ -196,7 +262,8 @@
196262 	if (align > CONFIG_CMA_ALIGNMENT)
197263 		align = CONFIG_CMA_ALIGNMENT;
198264 
199
-	return cma_alloc(dev_get_cma_area(dev), count, align, no_warn);
265
+	return cma_alloc(dev_get_cma_area(dev), count, align, GFP_KERNEL |
266
+			(no_warn ? __GFP_NOWARN : 0));
200267 }
201268 
202269 /**
....@@ -215,6 +282,102 @@
215282 	return cma_release(dev_get_cma_area(dev), pages, count);
216283 }
217284 
285
+static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
286
+{
287
+	unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT);
288
+
289
+	return cma_alloc(cma, size >> PAGE_SHIFT, align,
290
+				GFP_KERNEL | (gfp & __GFP_NOWARN));
291
+}
292
+
293
+/**
294
+ * dma_alloc_contiguous() - allocate contiguous pages
295
+ * @dev:   Pointer to device for which the allocation is performed.
296
+ * @size:  Requested allocation size.
297
+ * @gfp:   Allocation flags.
298
+ *
299
+ * tries to use device specific contiguous memory area if available, or it
300
+ * tries to use per-numa cma, if the allocation fails, it will fallback to
301
+ * try default global one.
302
+ *
303
+ * Note that it bypass one-page size of allocations from the per-numa and
304
+ * global area as the addresses within one page are always contiguous, so
305
+ * there is no need to waste CMA pages for that kind; it also helps reduce
306
+ * fragmentations.
307
+ */
308
+struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
309
+{
310
+#ifdef CONFIG_DMA_PERNUMA_CMA
311
+	int nid = dev_to_node(dev);
312
+#endif
313
+	bool allow_subpage_alloc = false;
314
+
315
+	/* CMA can be used only in the context which permits sleeping */
316
+	if (!gfpflags_allow_blocking(gfp))
317
+		return NULL;
318
+	if (dev->cma_area)
319
+		return cma_alloc_aligned(dev->cma_area, size, gfp);
320
+
321
+	if (size <= PAGE_SIZE) {
322
+		trace_android_vh_subpage_dma_contig_alloc(&allow_subpage_alloc, dev, &size);
323
+		if (!allow_subpage_alloc)
324
+			return NULL;
325
+	}
326
+
327
+#ifdef CONFIG_DMA_PERNUMA_CMA
328
+	if (nid != NUMA_NO_NODE && !(gfp & (GFP_DMA | GFP_DMA32))) {
329
+		struct cma *cma = dma_contiguous_pernuma_area[nid];
330
+		struct page *page;
331
+
332
+		if (cma) {
333
+			page = cma_alloc_aligned(cma, size, gfp);
334
+			if (page)
335
+				return page;
336
+		}
337
+	}
338
+#endif
339
+	if (!dma_contiguous_default_area)
340
+		return NULL;
341
+
342
+	return cma_alloc_aligned(dma_contiguous_default_area, size, gfp);
343
+}
344
+
345
+/**
346
+ * dma_free_contiguous() - release allocated pages
347
+ * @dev:   Pointer to device for which the pages were allocated.
348
+ * @page:  Pointer to the allocated pages.
349
+ * @size:  Size of allocated pages.
350
+ *
351
+ * This function releases memory allocated by dma_alloc_contiguous(). As the
352
+ * cma_release returns false when provided pages do not belong to contiguous
353
+ * area and true otherwise, this function then does a fallback __free_pages()
354
+ * upon a false-return.
355
+ */
356
+void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
357
+{
358
+	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
359
+
360
+	/* if dev has its own cma, free page from there */
361
+	if (dev->cma_area) {
362
+		if (cma_release(dev->cma_area, page, count))
363
+			return;
364
+	} else {
365
+		/*
366
+		 * otherwise, page is from either per-numa cma or default cma
367
+		 */
368
+#ifdef CONFIG_DMA_PERNUMA_CMA
369
+		if (cma_release(dma_contiguous_pernuma_area[page_to_nid(page)],
370
+					page, count))
371
+			return;
372
+#endif
373
+		if (cma_release(dma_contiguous_default_area, page, count))
374
+			return;
375
+	}
376
+
377
+	/* not in any cma, free from buddy */
378
+	__free_pages(page, get_order(size));
379
+}
380
+
218381 /*
219382  * Support for reserved memory regions defined in device tree
220383  */
....@@ -228,14 +391,14 @@
228391 
229392 static int rmem_cma_device_init(struct reserved_mem *rmem, struct device *dev)
230393 {
231
-	dev_set_cma_area(dev, rmem->priv);
394
+	dev->cma_area = rmem->priv;
232395 	return 0;
233396 }
234397 
235398 static void rmem_cma_device_release(struct reserved_mem *rmem,
236399 				    struct device *dev)
237400 {
238
-	dev_set_cma_area(dev, NULL);
401
+	dev->cma_area = NULL;
239402 }
240403 
241404 static const struct reserved_mem_ops rmem_cma_ops = {
....@@ -248,8 +411,15 @@
248411 	phys_addr_t align = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order);
249412 	phys_addr_t mask = align - 1;
250413 	unsigned long node = rmem->fdt_node;
414
+	bool default_cma = of_get_flat_dt_prop(node, "linux,cma-default", NULL);
251415 	struct cma *cma;
252416 	int err;
417
+
418
+	if (size_cmdline != -1 && default_cma) {
419
+		pr_info("Reserved memory: bypass %s node, using cmdline CMA params instead\n",
420
+			rmem->name);
421
+		return -EBUSY;
422
+	}
253423 
254424 	if (!of_get_flat_dt_prop(node, "reusable", NULL) ||
255425 	    of_get_flat_dt_prop(node, "no-map", NULL))
....@@ -268,11 +438,8 @@
268438 	/* Architecture specific contiguous memory fixup. */
269439 	dma_contiguous_early_fixup(rmem->base, rmem->size);
270440 
271
-	if (of_get_flat_dt_prop(node, "linux,cma-default", NULL))
272
-		dma_contiguous_set_default(cma);
273
-
274
-	if (of_get_flat_dt_prop(node, "inactive", NULL))
275
-		set_cma_area_inactive(cma);
441
+	if (default_cma)
442
+		dma_contiguous_default_area = cma;
276443 
277444 	rmem->ops = &rmem_cma_ops;
278445 	rmem->priv = cma;