forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 10ebd8556b7990499c896a550e3d416b444211e6 
 kernel/mm/cma.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * Contiguous Memory Allocator
34  *
....@@ -9,11 +10,6 @@
910  *	Michal Nazarewicz <mina86@mina86.com>
1011  *	Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
1112  *	Joonsoo Kim <iamjoonsoo.kim@lge.com>
12
- *
13
- * This program is free software; you can redistribute it and/or
14
- * modify it under the terms of the GNU General Public License as
15
- * published by the Free Software Foundation; either version 2 of the
16
- * License or (at your optional) any later version of the license.
1713  */
1814 
1915 #define pr_fmt(fmt) "cma: " fmt
....@@ -28,6 +24,7 @@
2824 #include <linux/memblock.h>
2925 #include <linux/err.h>
3026 #include <linux/mm.h>
27
+#include <linux/module.h>
3128 #include <linux/mutex.h>
3229 #include <linux/sizes.h>
3330 #include <linux/slab.h>
....@@ -36,9 +33,17 @@
3633 #include <linux/highmem.h>
3734 #include <linux/io.h>
3835 #include <linux/kmemleak.h>
36
+#include <linux/sched.h>
37
+#include <linux/jiffies.h>
3938 #include <trace/events/cma.h>
4039 
40
+#undef CREATE_TRACE_POINTS
41
+#include <trace/hooks/mm.h>
42
+
4143 #include "cma.h"
44
+
45
+extern void lru_cache_disable(void);
46
+extern void lru_cache_enable(void);
4247 
4348 struct cma cma_areas[MAX_CMA_AREAS];
4449 unsigned cma_area_count;
....@@ -48,17 +53,15 @@
4853 {
4954 	return PFN_PHYS(cma->base_pfn);
5055 }
51
-EXPORT_SYMBOL(cma_get_base);
5256 
5357 unsigned long cma_get_size(const struct cma *cma)
5458 {
5559 	return cma->count << PAGE_SHIFT;
5660 }
57
-EXPORT_SYMBOL(cma_get_size);
5861 
5962 const char *cma_get_name(const struct cma *cma)
6063 {
61
-	return cma->name ? cma->name : "(undefined)";
64
+	return cma->name;
6265 }
6366 EXPORT_SYMBOL_GPL(cma_get_name);
6467 
....@@ -100,45 +103,35 @@
100103 	mutex_unlock(&cma->lock);
101104 }
102105 
103
-static int __init cma_activate_area(struct cma *cma)
106
+static void __init cma_activate_area(struct cma *cma)
104107 {
105
-	int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long);
106
-	unsigned long base_pfn = cma->base_pfn, pfn = base_pfn;
107
-	unsigned i = cma->count >> pageblock_order;
108
+	unsigned long base_pfn = cma->base_pfn, pfn;
108109 	struct zone *zone;
109110 
110
-	cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
111
+	cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);
112
+	if (!cma->bitmap)
113
+		goto out_error;
111114 
112
-	if (!cma->bitmap) {
113
-		cma->count = 0;
114
-		return -ENOMEM;
115
+	if (IS_ENABLED(CONFIG_CMA_INACTIVE))
116
+		goto out;
117
+	/*
118
+	 * alloc_contig_range() requires the pfn range specified to be in the
119
+	 * same zone. Simplify by forcing the entire CMA resv range to be in the
120
+	 * same zone.
121
+	 */
122
+	WARN_ON_ONCE(!pfn_valid(base_pfn));
123
+	zone = page_zone(pfn_to_page(base_pfn));
124
+	for (pfn = base_pfn + 1; pfn < base_pfn + cma->count; pfn++) {
125
+		WARN_ON_ONCE(!pfn_valid(pfn));
126
+		if (page_zone(pfn_to_page(pfn)) != zone)
127
+			goto not_in_zone;
115128 	}
116129 
117
-	if (cma->inactive)
118
-		goto done;
130
+	for (pfn = base_pfn; pfn < base_pfn + cma->count;
131
+	     pfn += pageblock_nr_pages)
132
+		init_cma_reserved_pageblock(pfn_to_page(pfn));
119133 
120
-	WARN_ON_ONCE(!pfn_valid(pfn));
121
-	zone = page_zone(pfn_to_page(pfn));
122
-
123
-	do {
124
-		unsigned j;
125
-
126
-		base_pfn = pfn;
127
-		for (j = pageblock_nr_pages; j; --j, pfn++) {
128
-			WARN_ON_ONCE(!pfn_valid(pfn));
129
-			/*
130
-			 * alloc_contig_range requires the pfn range
131
-			 * specified to be in the same zone. Make this
132
-			 * simple by forcing the entire CMA resv range
133
-			 * to be in the same zone.
134
-			 */
135
-			if (page_zone(pfn_to_page(pfn)) != zone)
136
-				goto not_in_zone;
137
-		}
138
-		init_cma_reserved_pageblock(pfn_to_page(base_pfn));
139
-	} while (--i);
140
-
141
-done:
134
+out:
142135 	mutex_init(&cma->lock);
143136 
144137 #ifdef CONFIG_CMA_DEBUGFS
....@@ -146,25 +139,26 @@
146139 	spin_lock_init(&cma->mem_head_lock);
147140 #endif
148141 
149
-	return 0;
142
+	return;
150143 
151144 not_in_zone:
152
-	pr_err("CMA area %s could not be activated\n", cma->name);
153
-	kfree(cma->bitmap);
145
+	bitmap_free(cma->bitmap);
146
+out_error:
147
+	/* Expose all pages to the buddy, they are useless for CMA. */
148
+	for (pfn = base_pfn; pfn < base_pfn + cma->count; pfn++)
149
+		free_reserved_page(pfn_to_page(pfn));
150
+	totalcma_pages -= cma->count;
154151 	cma->count = 0;
155
-	return -EINVAL;
152
+	pr_err("CMA area %s could not be activated\n", cma->name);
153
+	return;
156154 }
157155 
158156 static int __init cma_init_reserved_areas(void)
159157 {
160158 	int i;
161159 
162
-	for (i = 0; i < cma_area_count; i++) {
163
-		int ret = cma_activate_area(&cma_areas[i]);
164
-
165
-		if (ret)
166
-			return ret;
167
-	}
160
+	for (i = 0; i < cma_area_count; i++)
161
+		cma_activate_area(&cma_areas[i]);
168162 
169163 	return 0;
170164 }
....@@ -188,7 +182,9 @@
188182 				 struct cma **res_cma)
189183 {
190184 	struct cma *cma;
185
+#if !IS_ENABLED(CONFIG_CMA_INACTIVE)
191186 	phys_addr_t alignment;
187
+#endif
192188 
193189 	/* Sanity checks */
194190 	if (cma_area_count == ARRAY_SIZE(cma_areas)) {
....@@ -199,6 +195,7 @@
199195 	if (!size || !memblock_is_region_reserved(base, size))
200196 		return -EINVAL;
201197 
198
+#if !IS_ENABLED(CONFIG_CMA_INACTIVE)
202199 	/* ensure minimal alignment required by mm core */
203200 	alignment = PAGE_SIZE <<
204201 			max_t(unsigned long, MAX_ORDER - 1, pageblock_order);
....@@ -209,19 +206,19 @@
209206 
210207 	if (ALIGN(base, alignment) != base || ALIGN(size, alignment) != size)
211208 		return -EINVAL;
209
+#endif
212210 
213211 	/*
214212 	 * Each reserved area must be initialised later, when more kernel
215213 	 * subsystems (like slab allocator) are available.
216214 	 */
217215 	cma = &cma_areas[cma_area_count];
218
-	if (name) {
219
-		cma->name = name;
220
-	} else {
221
-		cma->name = kasprintf(GFP_KERNEL, "cma%d\n", cma_area_count);
222
-		if (!cma->name)
223
-			return -ENOMEM;
224
-	}
216
+
217
+	if (name)
218
+		snprintf(cma->name, CMA_MAX_NAME, name);
219
+	else
220
+		snprintf(cma->name, CMA_MAX_NAME,  "cma%d\n", cma_area_count);
221
+
225222 	cma->base_pfn = PFN_DOWN(base);
226223 	cma->count = size >> PAGE_SHIFT;
227224 	cma->order_per_bit = order_per_bit;
....@@ -233,7 +230,7 @@
233230 }
234231 
235232 /**
236
- * cma_declare_contiguous() - reserve custom contiguous area
233
+ * cma_declare_contiguous_nid() - reserve custom contiguous area
237234  * @base: Base address of the reserved area optional, use 0 for any
238235  * @size: Size of the reserved area (in bytes),
239236  * @limit: End address of the reserved memory (optional, 0 for any).
....@@ -242,6 +239,7 @@
242239  * @fixed: hint about where to place the reserved area
243240  * @name: The name of the area. See function cma_init_reserved_mem()
244241  * @res_cma: Pointer to store the created cma region.
242
+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
245243  *
246244  * This function reserves memory from early allocator. It should be
247245  * called by arch specific code once the early allocator (memblock or bootmem)
....@@ -251,10 +249,11 @@
251249  * If @fixed is true, reserve contiguous area at exactly @base.  If false,
252250  * reserve in range from @base to @limit.
253251  */
254
-int __init cma_declare_contiguous(phys_addr_t base,
252
+int __init cma_declare_contiguous_nid(phys_addr_t base,
255253 			phys_addr_t size, phys_addr_t limit,
256254 			phys_addr_t alignment, unsigned int order_per_bit,
257
-			bool fixed, const char *name, struct cma **res_cma)
255
+			bool fixed, const char *name, struct cma **res_cma,
256
+			int nid)
258257 {
259258 	phys_addr_t memblock_end = memblock_end_of_DRAM();
260259 	phys_addr_t highmem_start;
....@@ -281,6 +280,7 @@
281280 	if (alignment && !is_power_of_2(alignment))
282281 		return -EINVAL;
283282 
283
+#if !IS_ENABLED(CONFIG_CMA_INACTIVE)
284284 	/*
285285 	 * Sanitise input arguments.
286286 	 * Pages both ends in CMA area could be merged into adjacent unmovable
....@@ -295,6 +295,7 @@
295295 			&base, &alignment);
296296 		goto err;
297297 	}
298
+#endif
298299 	base = ALIGN(base, alignment);
299300 	size = ALIGN(size, alignment);
300301 	limit &= ~(alignment - 1);
....@@ -349,16 +350,31 @@
349350 		 * memory in case of failure.
350351 		 */
351352 		if (base < highmem_start && limit > highmem_start) {
352
-			addr = memblock_alloc_range(size, alignment,
353
-						    highmem_start, limit,
354
-						    MEMBLOCK_NONE);
353
+			addr = memblock_alloc_range_nid(size, alignment,
354
+					highmem_start, limit, nid, true);
355355 			limit = highmem_start;
356356 		}
357357 
358
+		/*
359
+		 * If there is enough memory, try a bottom-up allocation first.
360
+		 * It will place the new cma area close to the start of the node
361
+		 * and guarantee that the compaction is moving pages out of the
362
+		 * cma area and not into it.
363
+		 * Avoid using first 4GB to not interfere with constrained zones
364
+		 * like DMA/DMA32.
365
+		 */
366
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
367
+		if (!memblock_bottom_up() && memblock_end >= SZ_4G + size) {
368
+			memblock_set_bottom_up(true);
369
+			addr = memblock_alloc_range_nid(size, alignment, SZ_4G,
370
+							limit, nid, true);
371
+			memblock_set_bottom_up(false);
372
+		}
373
+#endif
374
+
358375 		if (!addr) {
359
-			addr = memblock_alloc_range(size, alignment, base,
360
-						    limit,
361
-						    MEMBLOCK_NONE);
376
+			addr = memblock_alloc_range_nid(size, alignment, base,
377
+					limit, nid, true);
362378 			if (!addr) {
363379 				ret = -ENOMEM;
364380 				goto err;
....@@ -377,14 +393,23 @@
377393 	if (ret)
378394 		goto free_mem;
379395 
396
+#if !IS_ENABLED(CONFIG_CMA_INACTIVE)
380397 	pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size / SZ_1M,
381398 		&base);
399
+#else
400
+	pr_info("Reserved %ld KiB at %pa\n", (unsigned long)size / SZ_1K,
401
+		&base);
402
+#endif
382403 	return 0;
383404 
384405 free_mem:
385406 	memblock_free(base, size);
386407 err:
408
+#if !IS_ENABLED(CONFIG_CMA_INACTIVE)
387409 	pr_err("Failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M);
410
+#else
411
+	pr_err("Failed to reserve %ld KiB\n", (unsigned long)size / SZ_1K);
412
+#endif
388413 	return ret;
389414 }
390415 
....@@ -422,13 +447,13 @@
422447  * @cma:   Contiguous memory region for which the allocation is performed.
423448  * @count: Requested number of pages.
424449  * @align: Requested alignment of pages (in PAGE_SIZE order).
425
- * @no_warn: Avoid printing message about failed allocation
450
+ * @gfp_mask: GFP mask to use during the cma allocation.
426451  *
427452  * This function allocates part of contiguous memory on specific
428453  * contiguous memory area.
429454  */
430455 struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align,
431
-		       bool no_warn)
456
+		       gfp_t gfp_mask)
432457 {
433458 	unsigned long mask, offset;
434459 	unsigned long pfn = -1;
....@@ -437,15 +462,23 @@
437462 	size_t i;
438463 	struct page *page = NULL;
439464 	int ret = -ENOMEM;
465
+	int num_attempts = 0;
466
+	int max_retries = 5;
467
+	s64 ts;
468
+	struct cma_alloc_info cma_info = {0};
440469 
441
-	if (!cma || !cma->count)
442
-		return NULL;
470
+	trace_android_vh_cma_alloc_start(&ts);
443471 
444
-	pr_debug("%s(cma %p, count %zu, align %d)\n", __func__, (void *)cma,
445
-		 count, align);
472
+	if (!cma || !cma->count || !cma->bitmap)
473
+		goto out;
474
+
475
+	pr_debug("%s(cma %p, count %zu, align %d gfp_mask 0x%x)\n", __func__,
476
+			(void *)cma, count, align, gfp_mask);
446477 
447478 	if (!count)
448
-		return NULL;
479
+		goto out;
480
+
481
+	trace_cma_alloc_start(cma->name, count, align);
449482 
450483 	mask = cma_bitmap_aligned_mask(cma, align);
451484 	offset = cma_bitmap_aligned_offset(cma, align);
....@@ -453,16 +486,40 @@
453486 	bitmap_count = cma_bitmap_pages_to_bits(cma, count);
454487 
455488 	if (bitmap_count > bitmap_maxno)
456
-		return NULL;
489
+		goto out;
457490 
491
+	lru_cache_disable();
458492 	for (;;) {
493
+		struct acr_info info = {0};
494
+
459495 		mutex_lock(&cma->lock);
460496 		bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,
461497 				bitmap_maxno, start, bitmap_count, mask,
462498 				offset);
463499 		if (bitmap_no >= bitmap_maxno) {
464
-			mutex_unlock(&cma->lock);
465
-			break;
500
+			if ((num_attempts < max_retries) && (ret == -EBUSY)) {
501
+				mutex_unlock(&cma->lock);
502
+
503
+				if (fatal_signal_pending(current) ||
504
+				    (gfp_mask & __GFP_NORETRY))
505
+					break;
506
+
507
+				/*
508
+				 * Page may be momentarily pinned by some other
509
+				 * process which has been scheduled out, e.g.
510
+				 * in exit path, during unmap call, or process
511
+				 * fork and so cannot be freed there. Sleep
512
+				 * for 100ms and retry the allocation.
513
+				 */
514
+				start = 0;
515
+				ret = -ENOMEM;
516
+				schedule_timeout_killable(msecs_to_jiffies(100));
517
+				num_attempts++;
518
+				continue;
519
+			} else {
520
+				mutex_unlock(&cma->lock);
521
+				break;
522
+			}
466523 		}
467524 		bitmap_set(cma->bitmap, bitmap_no, bitmap_count);
468525 		/*
....@@ -473,15 +530,25 @@
473530 		mutex_unlock(&cma->lock);
474531 
475532 		pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
476
-		if (cma->inactive) {
477
-			ret = 0;
533
+		if (IS_ENABLED(CONFIG_CMA_INACTIVE)) {
478534 			page = pfn_to_page(pfn);
479
-			break;
535
+			lru_cache_enable();
536
+			goto out;
480537 		}
481538 		mutex_lock(&cma_mutex);
482
-		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA,
483
-				     GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
539
+		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask, &info);
484540 		mutex_unlock(&cma_mutex);
541
+		cma_info.nr_migrated += info.nr_migrated;
542
+		cma_info.nr_reclaimed += info.nr_reclaimed;
543
+		cma_info.nr_mapped += info.nr_mapped;
544
+		if (info.err) {
545
+			if (info.err & ACR_ERR_ISOLATE)
546
+				cma_info.nr_isolate_fail++;
547
+			if (info.err & ACR_ERR_MIGRATE)
548
+				cma_info.nr_migrate_fail++;
549
+			if (info.err & ACR_ERR_TEST)
550
+				cma_info.nr_test_fail++;
551
+		}
485552 		if (ret == 0) {
486553 			page = pfn_to_page(pfn);
487554 			break;
....@@ -493,11 +560,24 @@
493560 
494561 		pr_debug("%s(): memory range at %p is busy, retrying\n",
495562 			 __func__, pfn_to_page(pfn));
496
-		/* try again with a bit different memory target */
497
-		start = bitmap_no + mask + 1;
563
+
564
+		trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn),
565
+					   count, align);
566
+
567
+		if (info.failed_pfn && gfp_mask & __GFP_NORETRY) {
568
+			/* try again from following failed page */
569
+			start = (pfn_max_align_up(info.failed_pfn + 1) -
570
+				 cma->base_pfn) >> cma->order_per_bit;
571
+
572
+		} else {
573
+			/* try again with a bit different memory target */
574
+			start = bitmap_no + mask + 1;
575
+		}
498576 	}
499577 
500
-	trace_cma_alloc(pfn, page, count, align);
578
+	lru_cache_enable();
579
+	trace_cma_alloc_finish(cma->name, pfn, page, count, align);
580
+	trace_cma_alloc_info(cma->name, page, count, align, &cma_info);
501581 
502582 	/*
503583 	 * CMA can allocate multiple page blocks, which results in different
....@@ -509,13 +589,24 @@
509589 			page_kasan_tag_reset(page + i);
510590 	}
511591 
512
-	if (ret && !no_warn) {
513
-		pr_err("%s: alloc failed, req-size: %zu pages, ret: %d\n",
514
-			__func__, count, ret);
592
+	if (ret && !(gfp_mask & __GFP_NOWARN)) {
593
+		pr_err("%s: %s: alloc failed, req-size: %zu pages, ret: %d\n",
594
+		       __func__, cma->name, count, ret);
515595 		cma_debug_show_areas(cma);
516596 	}
517597 
518598 	pr_debug("%s(): returned %p\n", __func__, page);
599
+out:
600
+	trace_android_vh_cma_alloc_finish(cma, page, count, align, gfp_mask, ts);
601
+	if (page) {
602
+		count_vm_event(CMA_ALLOC_SUCCESS);
603
+		cma_sysfs_account_success_pages(cma, count);
604
+	} else {
605
+		count_vm_event(CMA_ALLOC_FAIL);
606
+		if (cma)
607
+			cma_sysfs_account_fail_pages(cma, count);
608
+	}
609
+
519610 	return page;
520611 }
521612 EXPORT_SYMBOL_GPL(cma_alloc);
....@@ -526,7 +617,7 @@
526617  * @pages: Allocated pages.
527618  * @count: Number of allocated pages.
528619  *
529
- * This function releases memory allocated by alloc_cma().
620
+ * This function releases memory allocated by cma_alloc().
530621  * It returns false when provided pages do not belong to contiguous area and
531622  * true otherwise.
532623  */
....@@ -537,7 +628,7 @@
537628 	if (!cma || !pages)
538629 		return false;
539630 
540
-	pr_debug("%s(page %p)\n", __func__, (void *)pages);
631
+	pr_debug("%s(page %p, count %u)\n", __func__, (void *)pages, count);
541632 
542633 	pfn = page_to_pfn(pages);
543634 
....@@ -545,16 +636,16 @@
545636 		return false;
546637 
547638 	VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);
548
-
549
-	if (!cma->inactive)
639
+	if (!IS_ENABLED(CONFIG_CMA_INACTIVE))
550640 		free_contig_range(pfn, count);
551641 	cma_clear_bitmap(cma, pfn, count);
552
-	trace_cma_release(pfn, pages, count);
642
+	trace_cma_release(cma->name, pfn, pages, count);
553643 
554644 	return true;
555645 }
556646 EXPORT_SYMBOL_GPL(cma_release);
557647 
648
+#ifdef CONFIG_NO_GKI
558649 unsigned long cma_used_pages(void)
559650 {
560651 	struct cma *cma;
....@@ -572,6 +663,7 @@
572663 	return val;
573664 }
574665 EXPORT_SYMBOL_GPL(cma_used_pages);
666
+#endif
575667 
576668 int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
577669 {
....@@ -587,9 +679,3 @@
587679 	return 0;
588680 }
589681 EXPORT_SYMBOL_GPL(cma_for_each_area);
590
-
591
-void set_cma_area_inactive(struct cma *cma)
592
-{
593
-	cma->inactive = true;
594
-}
595
-EXPORT_SYMBOL_GPL(set_cma_area_inactive);