forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/mm/memblock.c
....@@ -1,13 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-or-later
12 /*
23  * Procedures for maintaining information about logical memory blocks.
34  *
45  * Peter Bergner, IBM Corp.	June 2001.
56  * Copyright (C) 2001 Peter Bergner.
6
- *
7
- *      This program is free software; you can redistribute it and/or
8
- *      modify it under the terms of the GNU General Public License
9
- *      as published by the Free Software Foundation; either version
10
- *      2 of the License, or (at your option) any later version.
117  */
128 
139 #include <linux/kernel.h>
....@@ -20,12 +16,18 @@
2016 #include <linux/kmemleak.h>
2117 #include <linux/seq_file.h>
2218 #include <linux/memblock.h>
23
-#include <linux/bootmem.h>
2419 
2520 #include <asm/sections.h>
2621 #include <linux/io.h>
2722 
2823 #include "internal.h"
24
+
25
+#define INIT_MEMBLOCK_REGIONS			128
26
+#define INIT_PHYSMEM_REGIONS			4
27
+
28
+#ifndef INIT_MEMBLOCK_RESERVED_REGIONS
29
+# define INIT_MEMBLOCK_RESERVED_REGIONS		INIT_MEMBLOCK_REGIONS
30
+#endif
2931 
3032 /**
3133  * DOC: memblock overview
....@@ -42,50 +44,88 @@
4244  *   in the system, for instance when the memory is restricted with
4345  *   ``mem=`` command line parameter
4446  * * ``reserved`` - describes the regions that were allocated
45
- * * ``physmap`` - describes the actual physical memory regardless of
46
- *   the possible restrictions; the ``physmap`` type is only available
47
- *   on some architectures.
47
+ * * ``physmem`` - describes the actual physical memory available during
48
+ *   boot regardless of the possible restrictions and memory hot(un)plug;
49
+ *   the ``physmem`` type is only available on some architectures.
4850  *
49
- * Each region is represented by :c:type:`struct memblock_region` that
51
+ * Each region is represented by struct memblock_region that
5052  * defines the region extents, its attributes and NUMA node id on NUMA
51
- * systems. Every memory type is described by the :c:type:`struct
52
- * memblock_type` which contains an array of memory regions along with
53
- * the allocator metadata. The memory types are nicely wrapped with
54
- * :c:type:`struct memblock`. This structure is statically initialzed
55
- * at build time. The region arrays for the "memory" and "reserved"
56
- * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
57
- * "physmap" type to %INIT_PHYSMEM_REGIONS.
58
- * The :c:func:`memblock_allow_resize` enables automatic resizing of
59
- * the region arrays during addition of new regions. This feature
60
- * should be used with care so that memory allocated for the region
61
- * array will not overlap with areas that should be reserved, for
62
- * example initrd.
53
+ * systems. Every memory type is described by the struct memblock_type
54
+ * which contains an array of memory regions along with
55
+ * the allocator metadata. The "memory" and "reserved" types are nicely
56
+ * wrapped with struct memblock. This structure is statically
57
+ * initialized at build time. The region arrays are initially sized to
58
+ * %INIT_MEMBLOCK_REGIONS for "memory" and %INIT_MEMBLOCK_RESERVED_REGIONS
59
+ * for "reserved". The region array for "physmem" is initially sized to
60
+ * %INIT_PHYSMEM_REGIONS.
61
+ * The memblock_allow_resize() enables automatic resizing of the region
62
+ * arrays during addition of new regions. This feature should be used
63
+ * with care so that memory allocated for the region array will not
64
+ * overlap with areas that should be reserved, for example initrd.
6365  *
6466  * The early architecture setup should tell memblock what the physical
65
- * memory layout is by using :c:func:`memblock_add` or
66
- * :c:func:`memblock_add_node` functions. The first function does not
67
- * assign the region to a NUMA node and it is appropriate for UMA
68
- * systems. Yet, it is possible to use it on NUMA systems as well and
69
- * assign the region to a NUMA node later in the setup process using
70
- * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node`
71
- * performs such an assignment directly.
67
+ * memory layout is by using memblock_add() or memblock_add_node()
68
+ * functions. The first function does not assign the region to a NUMA
69
+ * node and it is appropriate for UMA systems. Yet, it is possible to
70
+ * use it on NUMA systems as well and assign the region to a NUMA node
71
+ * later in the setup process using memblock_set_node(). The
72
+ * memblock_add_node() performs such an assignment directly.
7273  *
73
- * Once memblock is setup the memory can be allocated using either
74
- * memblock or bootmem APIs.
74
+ * Once memblock is setup the memory can be allocated using one of the
75
+ * API variants:
7576  *
76
- * As the system boot progresses, the architecture specific
77
- * :c:func:`mem_init` function frees all the memory to the buddy page
78
- * allocator.
77
+ * * memblock_phys_alloc*() - these functions return the **physical**
78
+ *   address of the allocated memory
79
+ * * memblock_alloc*() - these functions return the **virtual** address
80
+ *   of the allocated memory.
7981  *
80
- * If an architecure enables %CONFIG_ARCH_DISCARD_MEMBLOCK, the
81
- * memblock data structures will be discarded after the system
82
- * initialization compltes.
82
+ * Note, that both API variants use implicit assumptions about allowed
83
+ * memory ranges and the fallback methods. Consult the documentation
84
+ * of memblock_alloc_internal() and memblock_alloc_range_nid()
85
+ * functions for more elaborate description.
86
+ *
87
+ * As the system boot progresses, the architecture specific mem_init()
88
+ * function frees all the memory to the buddy page allocator.
89
+ *
90
+ * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
91
+ * memblock data structures (except "physmem") will be discarded after the
92
+ * system initialization completes.
8393  */
8494 
95
+#ifndef CONFIG_NEED_MULTIPLE_NODES
96
+struct pglist_data __refdata contig_page_data;
97
+EXPORT_SYMBOL(contig_page_data);
98
+#endif
99
+
100
+#if defined(CONFIG_ROCKCHIP_THUNDER_BOOT) && defined(CONFIG_SMP)
101
+static unsigned long defer_start __initdata;
102
+static unsigned long defer_end __initdata;
103
+
104
+#define DEFAULT_DEFER_FREE_BLOCK_SIZE SZ_256M
105
+static unsigned long defer_free_block_size __initdata =
106
+	DEFAULT_DEFER_FREE_BLOCK_SIZE;
107
+
108
+static int __init early_defer_free_block_size(char *p)
109
+{
110
+	defer_free_block_size = memparse(p, &p);
111
+
112
+	pr_debug("defer_free_block_size = 0x%lx\n", defer_free_block_size);
113
+
114
+	return 0;
115
+}
116
+
117
+early_param("defer_free_block_size", early_defer_free_block_size);
118
+#endif
119
+
120
+unsigned long max_low_pfn;
121
+unsigned long min_low_pfn;
122
+unsigned long max_pfn;
123
+unsigned long long max_possible_pfn;
124
+
85125 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
86
-static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
126
+static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
87127 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
88
-static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS] __initdata_memblock;
128
+static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
89129 #endif
90130 
91131 struct memblock memblock __initdata_memblock = {
....@@ -96,27 +136,49 @@
96136 
97137 	.reserved.regions	= memblock_reserved_init_regions,
98138 	.reserved.cnt		= 1,	/* empty dummy entry */
99
-	.reserved.max		= INIT_MEMBLOCK_REGIONS,
139
+	.reserved.max		= INIT_MEMBLOCK_RESERVED_REGIONS,
100140 	.reserved.name		= "reserved",
101
-
102
-#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
103
-	.physmem.regions	= memblock_physmem_init_regions,
104
-	.physmem.cnt		= 1,	/* empty dummy entry */
105
-	.physmem.max		= INIT_PHYSMEM_REGIONS,
106
-	.physmem.name		= "physmem",
107
-#endif
108141 
109142 	.bottom_up		= false,
110143 	.current_limit		= MEMBLOCK_ALLOC_ANYWHERE,
111144 };
112145 
113
-int memblock_debug __initdata_memblock;
146
+#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
147
+struct memblock_type physmem = {
148
+	.regions		= memblock_physmem_init_regions,
149
+	.cnt			= 1,	/* empty dummy entry */
150
+	.max			= INIT_PHYSMEM_REGIONS,
151
+	.name			= "physmem",
152
+};
153
+#endif
154
+
155
+/*
156
+ * keep a pointer to &memblock.memory in the text section to use it in
157
+ * __next_mem_range() and its helpers.
158
+ *  For architectures that do not keep memblock data after init, this
159
+ * pointer will be reset to NULL at memblock_discard()
160
+ */
161
+static __refdata struct memblock_type *memblock_memory = &memblock.memory;
162
+
163
+#define for_each_memblock_type(i, memblock_type, rgn)			\
164
+	for (i = 0, rgn = &memblock_type->regions[0];			\
165
+	     i < memblock_type->cnt;					\
166
+	     i++, rgn = &memblock_type->regions[i])
167
+
168
+#define memblock_dbg(fmt, ...)						\
169
+	do {								\
170
+		if (memblock_debug)					\
171
+			pr_info(fmt, ##__VA_ARGS__);			\
172
+	} while (0)
173
+
174
+static int memblock_debug __initdata_memblock;
175
+static bool memblock_nomap_remove __initdata_memblock;
114176 static bool system_has_some_mirror __initdata_memblock = false;
115177 static int memblock_can_resize __initdata_memblock;
116178 static int memblock_memory_in_slab __initdata_memblock = 0;
117179 static int memblock_reserved_in_slab __initdata_memblock = 0;
118180 
119
-enum memblock_flags __init_memblock choose_memblock_flags(void)
181
+static enum memblock_flags __init_memblock choose_memblock_flags(void)
120182 {
121183 	return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
122184 }
....@@ -140,6 +202,8 @@
140202 					phys_addr_t base, phys_addr_t size)
141203 {
142204 	unsigned long i;
205
+
206
+	memblock_cap_size(base, &size);
143207 
144208 	for (i = 0; i < type->cnt; i++)
145209 		if (memblock_addrs_overlap(base, size, type->regions[i].base,
....@@ -237,7 +301,7 @@
237301  * Return:
238302  * Found address on success, 0 on failure.
239303  */
240
-phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
304
+static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
241305 					phys_addr_t align, phys_addr_t start,
242306 					phys_addr_t end, int nid,
243307 					enum memblock_flags flags)
....@@ -311,7 +375,7 @@
311375 	}
312376 }
313377 
314
-#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
378
+#ifndef CONFIG_ARCH_KEEP_MEMBLOCK
315379 /**
316380  * memblock_discard - discard memory and reserved arrays if they were allocated
317381  */
....@@ -338,6 +402,8 @@
338402 		else
339403 			__memblock_free_late(addr, size);
340404 	}
405
+
406
+	memblock_memory = NULL;
341407 }
342408 #endif
343409 
....@@ -388,17 +454,7 @@
388454 	else
389455 		in_slab = &memblock_reserved_in_slab;
390456 
391
-	/* Try to find some space for it.
392
-	 *
393
-	 * WARNING: We assume that either slab_is_available() and we use it or
394
-	 * we use MEMBLOCK for allocations. That means that this is unsafe to
395
-	 * use when bootmem is currently active (unless bootmem itself is
396
-	 * implemented on top of MEMBLOCK which isn't the case yet)
397
-	 *
398
-	 * This should however not be an issue for now, as we currently only
399
-	 * call into MEMBLOCK while it's still active, or much later when slab
400
-	 * is active for memory hotplug operations
401
-	 */
457
+	/* Try to find some space for it */
402458 	if (use_slab) {
403459 		new_array = kmalloc(new_size, GFP_KERNEL);
404460 		addr = new_array ? __pa(new_array) : 0;
....@@ -535,7 +591,7 @@
535591  * Return:
536592  * 0 on success, -errno on failure.
537593  */
538
-int __init_memblock memblock_add_range(struct memblock_type *type,
594
+static int __init_memblock memblock_add_range(struct memblock_type *type,
539595 				phys_addr_t base, phys_addr_t size,
540596 				int nid, enum memblock_flags flags)
541597 {
....@@ -580,7 +636,7 @@
580636 		 * area, insert that portion.
581637 		 */
582638 		if (rbase > base) {
583
-#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
639
+#ifdef CONFIG_NEED_MULTIPLE_NODES
584640 			WARN_ON(nid != memblock_get_region_node(rgn));
585641 #endif
586642 			WARN_ON(flags != rgn->flags);
....@@ -654,7 +710,7 @@
654710 {
655711 	phys_addr_t end = base + size - 1;
656712 
657
-	memblock_dbg("memblock_add: [%pa-%pa] %pF\n",
713
+	memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
658714 		     &base, &end, (void *)_RET_IP_);
659715 
660716 	return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
....@@ -755,34 +811,55 @@
755811 {
756812 	phys_addr_t end = base + size - 1;
757813 
758
-	memblock_dbg("memblock_remove: [%pa-%pa] %pS\n",
814
+	memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
759815 		     &base, &end, (void *)_RET_IP_);
760816 
761817 	return memblock_remove_range(&memblock.memory, base, size);
762818 }
763819 
764
-
820
+/**
821
+ * memblock_free - free boot memory block
822
+ * @base: phys starting address of the  boot memory block
823
+ * @size: size of the boot memory block in bytes
824
+ *
825
+ * Free boot memory block previously allocated by memblock_alloc_xx() API.
826
+ * The freeing memory will not be released to the buddy allocator.
827
+ */
765828 int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
766829 {
767830 	phys_addr_t end = base + size - 1;
768831 
769
-	memblock_dbg("   memblock_free: [%pa-%pa] %pF\n",
832
+	memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
770833 		     &base, &end, (void *)_RET_IP_);
771834 
772835 	kmemleak_free_part_phys(base, size);
773836 	return memblock_remove_range(&memblock.reserved, base, size);
774837 }
838
+#ifdef CONFIG_ARCH_KEEP_MEMBLOCK
775839 EXPORT_SYMBOL_GPL(memblock_free);
840
+#endif
776841 
777842 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
778843 {
779844 	phys_addr_t end = base + size - 1;
780845 
781
-	memblock_dbg("memblock_reserve: [%pa-%pa] %pF\n",
846
+	memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
782847 		     &base, &end, (void *)_RET_IP_);
783848 
784849 	return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
785850 }
851
+
852
+#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
853
+int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
854
+{
855
+	phys_addr_t end = base + size - 1;
856
+
857
+	memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
858
+		     &base, &end, (void *)_RET_IP_);
859
+
860
+	return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
861
+}
862
+#endif
786863 
787864 /**
788865  * memblock_setclr_flag - set or clear flag for a memory region
....@@ -805,11 +882,14 @@
805882 	if (ret)
806883 		return ret;
807884 
808
-	for (i = start_rgn; i < end_rgn; i++)
885
+	for (i = start_rgn; i < end_rgn; i++) {
886
+		struct memblock_region *r = &type->regions[i];
887
+
809888 		if (set)
810
-			memblock_set_region_flags(&type->regions[i], flag);
889
+			r->flags |= flag;
811890 		else
812
-			memblock_clear_region_flags(&type->regions[i], flag);
891
+			r->flags &= ~flag;
892
+	}
813893 
814894 	memblock_merge_regions(type);
815895 	return 0;
....@@ -877,40 +957,38 @@
877957 	return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
878958 }
879959 
880
-/**
881
- * __next_reserved_mem_region - next function for for_each_reserved_region()
882
- * @idx: pointer to u64 loop variable
883
- * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL
884
- * @out_end: ptr to phys_addr_t for end address of the region, can be %NULL
885
- *
886
- * Iterate over all reserved memory regions.
887
- */
888
-void __init_memblock __next_reserved_mem_region(u64 *idx,
889
-					   phys_addr_t *out_start,
890
-					   phys_addr_t *out_end)
960
+static bool should_skip_region(struct memblock_type *type,
961
+			       struct memblock_region *m,
962
+			       int nid, int flags)
891963 {
892
-	struct memblock_type *type = &memblock.reserved;
964
+	int m_nid = memblock_get_region_node(m);
893965 
894
-	if (*idx < type->cnt) {
895
-		struct memblock_region *r = &type->regions[*idx];
896
-		phys_addr_t base = r->base;
897
-		phys_addr_t size = r->size;
966
+	/* we never skip regions when iterating memblock.reserved or physmem */
967
+	if (type != memblock_memory)
968
+		return false;
898969 
899
-		if (out_start)
900
-			*out_start = base;
901
-		if (out_end)
902
-			*out_end = base + size - 1;
970
+	/* only memory regions are associated with nodes, check it */
971
+	if (nid != NUMA_NO_NODE && nid != m_nid)
972
+		return true;
903973 
904
-		*idx += 1;
905
-		return;
906
-	}
974
+	/* skip hotpluggable memory regions if needed */
975
+	if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
976
+	    !(flags & MEMBLOCK_HOTPLUG))
977
+		return true;
907978 
908
-	/* signal end of iteration */
909
-	*idx = ULLONG_MAX;
979
+	/* if we want mirror memory skip non-mirror memory regions */
980
+	if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
981
+		return true;
982
+
983
+	/* skip nomap memory unless we were asked for it explicitly */
984
+	if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
985
+		return true;
986
+
987
+	return false;
910988 }
911989 
912990 /**
913
- * __next__mem_range - next function for for_each_free_mem_range() etc.
991
+ * __next_mem_range - next function for for_each_free_mem_range() etc.
914992  * @idx: pointer to u64 loop variable
915993  * @nid: node selector, %NUMA_NO_NODE for all nodes
916994  * @flags: pick from blocks based on memory attributes
....@@ -935,12 +1013,10 @@
9351013  * As both region arrays are sorted, the function advances the two indices
9361014  * in lockstep and returns each intersection.
9371015  */
938
-void __init_memblock __next_mem_range(u64 *idx, int nid,
939
-				      enum memblock_flags flags,
940
-				      struct memblock_type *type_a,
941
-				      struct memblock_type *type_b,
942
-				      phys_addr_t *out_start,
943
-				      phys_addr_t *out_end, int *out_nid)
1016
+void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
1017
+		      struct memblock_type *type_a,
1018
+		      struct memblock_type *type_b, phys_addr_t *out_start,
1019
+		      phys_addr_t *out_end, int *out_nid)
9441020 {
9451021 	int idx_a = *idx & 0xffffffff;
9461022 	int idx_b = *idx >> 32;
....@@ -956,20 +1032,7 @@
9561032 		phys_addr_t m_end = m->base + m->size;
9571033 		int	    m_nid = memblock_get_region_node(m);
9581034 
959
-		/* only memory regions are associated with nodes, check it */
960
-		if (nid != NUMA_NO_NODE && nid != m_nid)
961
-			continue;
962
-
963
-		/* skip hotpluggable memory regions if needed */
964
-		if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
965
-			continue;
966
-
967
-		/* if we want mirror memory skip non-mirror memory regions */
968
-		if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
969
-			continue;
970
-
971
-		/* skip nomap memory unless we were asked for it explicitly */
972
-		if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1035
+		if (should_skip_region(type_a, m, nid, flags))
9731036 			continue;
9741037 
9751038 		if (!type_b) {
....@@ -1073,20 +1136,7 @@
10731136 		phys_addr_t m_end = m->base + m->size;
10741137 		int m_nid = memblock_get_region_node(m);
10751138 
1076
-		/* only memory regions are associated with nodes, check it */
1077
-		if (nid != NUMA_NO_NODE && nid != m_nid)
1078
-			continue;
1079
-
1080
-		/* skip hotpluggable memory regions if needed */
1081
-		if (movable_node_is_enabled() && memblock_is_hotpluggable(m))
1082
-			continue;
1083
-
1084
-		/* if we want mirror memory skip non-mirror memory regions */
1085
-		if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
1086
-			continue;
1087
-
1088
-		/* skip nomap memory unless we were asked for it explicitly */
1089
-		if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1139
+		if (should_skip_region(type_a, m, nid, flags))
10901140 			continue;
10911141 
10921142 		if (!type_b) {
....@@ -1139,9 +1189,8 @@
11391189 	*idx = ULLONG_MAX;
11401190 }
11411191 
1142
-#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
11431192 /*
1144
- * Common iterator interface used to define for_each_mem_range().
1193
+ * Common iterator interface used to define for_each_mem_pfn_range().
11451194  */
11461195 void __init_memblock __next_mem_pfn_range(int *idx, int nid,
11471196 				unsigned long *out_start_pfn,
....@@ -1149,13 +1198,15 @@
11491198 {
11501199 	struct memblock_type *type = &memblock.memory;
11511200 	struct memblock_region *r;
1201
+	int r_nid;
11521202 
11531203 	while (++*idx < type->cnt) {
11541204 		r = &type->regions[*idx];
1205
+		r_nid = memblock_get_region_node(r);
11551206 
11561207 		if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
11571208 			continue;
1158
-		if (nid == MAX_NUMNODES || nid == r->nid)
1209
+		if (nid == MAX_NUMNODES || nid == r_nid)
11591210 			break;
11601211 	}
11611212 	if (*idx >= type->cnt) {
....@@ -1168,7 +1219,7 @@
11681219 	if (out_end_pfn)
11691220 		*out_end_pfn = PFN_DOWN(r->base + r->size);
11701221 	if (out_nid)
1171
-		*out_nid = r->nid;
1222
+		*out_nid = r_nid;
11721223 }
11731224 
11741225 /**
....@@ -1187,6 +1238,7 @@
11871238 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
11881239 				      struct memblock_type *type, int nid)
11891240 {
1241
+#ifdef CONFIG_NEED_MULTIPLE_NODES
11901242 	int start_rgn, end_rgn;
11911243 	int i, ret;
11921244 
....@@ -1198,167 +1250,129 @@
11981250 		memblock_set_region_node(&type->regions[i], nid);
11991251 
12001252 	memblock_merge_regions(type);
1201
-	return 0;
1202
-}
1203
-#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1204
-
1205
-static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1206
-					phys_addr_t align, phys_addr_t start,
1207
-					phys_addr_t end, int nid,
1208
-					enum memblock_flags flags)
1209
-{
1210
-	phys_addr_t found;
1211
-
1212
-	if (!align)
1213
-		align = SMP_CACHE_BYTES;
1214
-
1215
-	found = memblock_find_in_range_node(size, align, start, end, nid,
1216
-					    flags);
1217
-	if (found && !memblock_reserve(found, size)) {
1218
-		/*
1219
-		 * The min_count is set to 0 so that memblock allocations are
1220
-		 * never reported as leaks.
1221
-		 */
1222
-		kmemleak_alloc_phys(found, size, 0, 0);
1223
-		return found;
1224
-	}
1253
+#endif
12251254 	return 0;
12261255 }
12271256 
1228
-phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align,
1229
-					phys_addr_t start, phys_addr_t end,
1230
-					enum memblock_flags flags)
1231
-{
1232
-	return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1233
-					flags);
1234
-}
1235
-
1236
-phys_addr_t __init memblock_alloc_base_nid(phys_addr_t size,
1237
-					phys_addr_t align, phys_addr_t max_addr,
1238
-					int nid, enum memblock_flags flags)
1239
-{
1240
-	return memblock_alloc_range_nid(size, align, 0, max_addr, nid, flags);
1241
-}
1242
-
1243
-phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid)
1244
-{
1245
-	enum memblock_flags flags = choose_memblock_flags();
1246
-	phys_addr_t ret;
1247
-
1248
-again:
1249
-	ret = memblock_alloc_base_nid(size, align, MEMBLOCK_ALLOC_ACCESSIBLE,
1250
-				      nid, flags);
1251
-
1252
-	if (!ret && (flags & MEMBLOCK_MIRROR)) {
1253
-		flags &= ~MEMBLOCK_MIRROR;
1254
-		goto again;
1255
-	}
1256
-	return ret;
1257
-}
1258
-
1259
-phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
1260
-{
1261
-	return memblock_alloc_base_nid(size, align, max_addr, NUMA_NO_NODE,
1262
-				       MEMBLOCK_NONE);
1263
-}
1264
-
1265
-phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
1266
-{
1267
-	phys_addr_t alloc;
1268
-
1269
-	alloc = __memblock_alloc_base(size, align, max_addr);
1270
-
1271
-	if (alloc == 0)
1272
-		panic("ERROR: Failed to allocate %pa bytes below %pa.\n",
1273
-		      &size, &max_addr);
1274
-
1275
-	return alloc;
1276
-}
1277
-
1278
-phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align)
1279
-{
1280
-	return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
1281
-}
1282
-
1283
-phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
1284
-{
1285
-	phys_addr_t res = memblock_alloc_nid(size, align, nid);
1286
-
1287
-	if (res)
1288
-		return res;
1289
-	return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
1290
-}
1291
-
1292
-#if defined(CONFIG_NO_BOOTMEM)
1257
+#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
12931258 /**
1294
- * memblock_virt_alloc_internal - allocate boot memory block
1259
+ * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1260
+ *
1261
+ * @idx: pointer to u64 loop variable
1262
+ * @zone: zone in which all of the memory blocks reside
1263
+ * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1264
+ * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1265
+ *
1266
+ * This function is meant to be a zone/pfn specific wrapper for the
1267
+ * for_each_mem_range type iterators. Specifically they are used in the
1268
+ * deferred memory init routines and as such we were duplicating much of
1269
+ * this logic throughout the code. So instead of having it in multiple
1270
+ * locations it seemed like it would make more sense to centralize this to
1271
+ * one new iterator that does everything they need.
1272
+ */
1273
+void __init_memblock
1274
+__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1275
+			     unsigned long *out_spfn, unsigned long *out_epfn)
1276
+{
1277
+	int zone_nid = zone_to_nid(zone);
1278
+	phys_addr_t spa, epa;
1279
+	int nid;
1280
+
1281
+	__next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1282
+			 &memblock.memory, &memblock.reserved,
1283
+			 &spa, &epa, &nid);
1284
+
1285
+	while (*idx != U64_MAX) {
1286
+		unsigned long epfn = PFN_DOWN(epa);
1287
+		unsigned long spfn = PFN_UP(spa);
1288
+
1289
+		/*
1290
+		 * Verify the end is at least past the start of the zone and
1291
+		 * that we have at least one PFN to initialize.
1292
+		 */
1293
+		if (zone->zone_start_pfn < epfn && spfn < epfn) {
1294
+			/* if we went too far just stop searching */
1295
+			if (zone_end_pfn(zone) <= spfn) {
1296
+				*idx = U64_MAX;
1297
+				break;
1298
+			}
1299
+
1300
+			if (out_spfn)
1301
+				*out_spfn = max(zone->zone_start_pfn, spfn);
1302
+			if (out_epfn)
1303
+				*out_epfn = min(zone_end_pfn(zone), epfn);
1304
+
1305
+			return;
1306
+		}
1307
+
1308
+		__next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1309
+				 &memblock.memory, &memblock.reserved,
1310
+				 &spa, &epa, &nid);
1311
+	}
1312
+
1313
+	/* signal end of iteration */
1314
+	if (out_spfn)
1315
+		*out_spfn = ULONG_MAX;
1316
+	if (out_epfn)
1317
+		*out_epfn = 0;
1318
+}
1319
+
1320
+#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1321
+
1322
+/**
1323
+ * memblock_alloc_range_nid - allocate boot memory block
12951324  * @size: size of memory block to be allocated in bytes
12961325  * @align: alignment of the region and block's size
1297
- * @min_addr: the lower bound of the memory region to allocate (phys address)
1298
- * @max_addr: the upper bound of the memory region to allocate (phys address)
1326
+ * @start: the lower bound of the memory region to allocate (phys address)
1327
+ * @end: the upper bound of the memory region to allocate (phys address)
12991328  * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1300
- *
1301
- * The @min_addr limit is dropped if it can not be satisfied and the allocation
1302
- * will fall back to memory below @min_addr. Also, allocation may fall back
1303
- * to any node in the system if the specified node can not
1304
- * hold the requested memory.
1329
+ * @exact_nid: control the allocation fall back to other nodes
13051330  *
13061331  * The allocation is performed from memory region limited by
1307
- * memblock.current_limit if @max_addr == %BOOTMEM_ALLOC_ACCESSIBLE.
1332
+ * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
13081333  *
1309
- * The memory block is aligned on %SMP_CACHE_BYTES if @align == 0.
1334
+ * If the specified node can not hold the requested memory and @exact_nid
1335
+ * is false, the allocation falls back to any node in the system.
13101336  *
1311
- * The phys address of allocated boot memory block is converted to virtual and
1312
- * allocated memory is reset to 0.
1337
+ * For systems with memory mirroring, the allocation is attempted first
1338
+ * from the regions with mirroring enabled and then retried from any
1339
+ * memory region.
13131340  *
1314
- * In addition, function sets the min_count to 0 using kmemleak_alloc for
1341
+ * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
13151342  * allocated boot memory block, so that it is never reported as leaks.
13161343  *
13171344  * Return:
1318
- * Virtual address of allocated memory block on success, NULL on failure.
1345
+ * Physical address of allocated memory block on success, %0 on failure.
13191346  */
1320
-static void * __init memblock_virt_alloc_internal(
1321
-				phys_addr_t size, phys_addr_t align,
1322
-				phys_addr_t min_addr, phys_addr_t max_addr,
1323
-				int nid)
1347
+phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1348
+					phys_addr_t align, phys_addr_t start,
1349
+					phys_addr_t end, int nid,
1350
+					bool exact_nid)
13241351 {
1325
-	phys_addr_t alloc;
1326
-	void *ptr;
13271352 	enum memblock_flags flags = choose_memblock_flags();
1353
+	phys_addr_t found;
13281354 
13291355 	if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
13301356 		nid = NUMA_NO_NODE;
13311357 
1332
-	/*
1333
-	 * Detect any accidental use of these APIs after slab is ready, as at
1334
-	 * this moment memblock may be deinitialized already and its
1335
-	 * internal data may be destroyed (after execution of free_all_bootmem)
1336
-	 */
1337
-	if (WARN_ON_ONCE(slab_is_available()))
1338
-		return kzalloc_node(size, GFP_NOWAIT, nid);
1339
-
1340
-	if (!align)
1358
+	if (!align) {
1359
+		/* Can't use WARNs this early in boot on powerpc */
1360
+		dump_stack();
13411361 		align = SMP_CACHE_BYTES;
1342
-
1343
-	if (max_addr > memblock.current_limit)
1344
-		max_addr = memblock.current_limit;
1345
-again:
1346
-	alloc = memblock_find_in_range_node(size, align, min_addr, max_addr,
1347
-					    nid, flags);
1348
-	if (alloc && !memblock_reserve(alloc, size))
1349
-		goto done;
1350
-
1351
-	if (nid != NUMA_NO_NODE) {
1352
-		alloc = memblock_find_in_range_node(size, align, min_addr,
1353
-						    max_addr, NUMA_NO_NODE,
1354
-						    flags);
1355
-		if (alloc && !memblock_reserve(alloc, size))
1356
-			goto done;
13571362 	}
13581363 
1359
-	if (min_addr) {
1360
-		min_addr = 0;
1361
-		goto again;
1364
+again:
1365
+	found = memblock_find_in_range_node(size, align, start, end, nid,
1366
+					    flags);
1367
+	if (found && !memblock_reserve(found, size))
1368
+		goto done;
1369
+
1370
+	if (nid != NUMA_NO_NODE && !exact_nid) {
1371
+		found = memblock_find_in_range_node(size, align, start,
1372
+						    end, NUMA_NO_NODE,
1373
+						    flags);
1374
+		if (found && !memblock_reserve(found, size))
1375
+			goto done;
13621376 	}
13631377 
13641378 	if (flags & MEMBLOCK_MIRROR) {
....@@ -1368,32 +1382,163 @@
13681382 		goto again;
13691383 	}
13701384 
1371
-	return NULL;
1372
-done:
1373
-	ptr = phys_to_virt(alloc);
1385
+	return 0;
13741386 
1387
+done:
13751388 	/* Skip kmemleak for kasan_init() due to high volume. */
1376
-	if (max_addr != MEMBLOCK_ALLOC_KASAN)
1389
+	if (end != MEMBLOCK_ALLOC_KASAN)
13771390 		/*
1378
-		 * The min_count is set to 0 so that bootmem allocated
1391
+		 * The min_count is set to 0 so that memblock allocated
13791392 		 * blocks are never reported as leaks. This is because many
13801393 		 * of these blocks are only referred via the physical
13811394 		 * address which is not looked up by kmemleak.
13821395 		 */
1383
-		kmemleak_alloc(ptr, size, 0, 0);
1396
+		kmemleak_alloc_phys(found, size, 0, 0);
1397
+
1398
+	return found;
1399
+}
1400
+
1401
+/**
1402
+ * memblock_phys_alloc_range - allocate a memory block inside specified range
1403
+ * @size: size of memory block to be allocated in bytes
1404
+ * @align: alignment of the region and block's size
1405
+ * @start: the lower bound of the memory region to allocate (physical address)
1406
+ * @end: the upper bound of the memory region to allocate (physical address)
1407
+ *
1408
+ * Allocate @size bytes in the between @start and @end.
1409
+ *
1410
+ * Return: physical address of the allocated memory block on success,
1411
+ * %0 on failure.
1412
+ */
1413
+phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
1414
+					     phys_addr_t align,
1415
+					     phys_addr_t start,
1416
+					     phys_addr_t end)
1417
+{
1418
+	memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
1419
+		     __func__, (u64)size, (u64)align, &start, &end,
1420
+		     (void *)_RET_IP_);
1421
+	return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1422
+					false);
1423
+}
1424
+
1425
+/**
1426
+ * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node
1427
+ * @size: size of memory block to be allocated in bytes
1428
+ * @align: alignment of the region and block's size
1429
+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1430
+ *
1431
+ * Allocates memory block from the specified NUMA node. If the node
1432
+ * has no available memory, attempts to allocated from any node in the
1433
+ * system.
1434
+ *
1435
+ * Return: physical address of the allocated memory block on success,
1436
+ * %0 on failure.
1437
+ */
1438
+phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
1439
+{
1440
+	return memblock_alloc_range_nid(size, align, 0,
1441
+					MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
1442
+}
1443
+
1444
+/**
1445
+ * memblock_alloc_internal - allocate boot memory block
1446
+ * @size: size of memory block to be allocated in bytes
1447
+ * @align: alignment of the region and block's size
1448
+ * @min_addr: the lower bound of the memory region to allocate (phys address)
1449
+ * @max_addr: the upper bound of the memory region to allocate (phys address)
1450
+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1451
+ * @exact_nid: control the allocation fall back to other nodes
1452
+ *
1453
+ * Allocates memory block using memblock_alloc_range_nid() and
1454
+ * converts the returned physical address to virtual.
1455
+ *
1456
+ * The @min_addr limit is dropped if it can not be satisfied and the allocation
1457
+ * will fall back to memory below @min_addr. Other constraints, such
1458
+ * as node and mirrored memory will be handled again in
1459
+ * memblock_alloc_range_nid().
1460
+ *
1461
+ * Return:
1462
+ * Virtual address of allocated memory block on success, NULL on failure.
1463
+ */
1464
+static void * __init memblock_alloc_internal(
1465
+				phys_addr_t size, phys_addr_t align,
1466
+				phys_addr_t min_addr, phys_addr_t max_addr,
1467
+				int nid, bool exact_nid)
1468
+{
1469
+	phys_addr_t alloc;
1470
+
1471
+	/*
1472
+	 * Detect any accidental use of these APIs after slab is ready, as at
1473
+	 * this moment memblock may be deinitialized already and its
1474
+	 * internal data may be destroyed (after execution of memblock_free_all)
1475
+	 */
1476
+	if (WARN_ON_ONCE(slab_is_available()))
1477
+		return kzalloc_node(size, GFP_NOWAIT, nid);
1478
+
1479
+	if (max_addr > memblock.current_limit)
1480
+		max_addr = memblock.current_limit;
1481
+
1482
+	alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1483
+					exact_nid);
1484
+
1485
+	/* retry allocation without lower limit */
1486
+	if (!alloc && min_addr)
1487
+		alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1488
+						exact_nid);
1489
+
1490
+	if (!alloc)
1491
+		return NULL;
1492
+
1493
+	return phys_to_virt(alloc);
1494
+}
1495
+
1496
+/**
1497
+ * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1498
+ * without zeroing memory
1499
+ * @size: size of memory block to be allocated in bytes
1500
+ * @align: alignment of the region and block's size
1501
+ * @min_addr: the lower bound of the memory region from where the allocation
1502
+ *	  is preferred (phys address)
1503
+ * @max_addr: the upper bound of the memory region from where the allocation
1504
+ *	      is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1505
+ *	      allocate only from memory limited by memblock.current_limit value
1506
+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1507
+ *
1508
+ * Public function, provides additional debug information (including caller
1509
+ * info), if enabled. Does not zero allocated memory.
1510
+ *
1511
+ * Return:
1512
+ * Virtual address of allocated memory block on success, NULL on failure.
1513
+ */
1514
+void * __init memblock_alloc_exact_nid_raw(
1515
+			phys_addr_t size, phys_addr_t align,
1516
+			phys_addr_t min_addr, phys_addr_t max_addr,
1517
+			int nid)
1518
+{
1519
+	void *ptr;
1520
+
1521
+	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1522
+		     __func__, (u64)size, (u64)align, nid, &min_addr,
1523
+		     &max_addr, (void *)_RET_IP_);
1524
+
1525
+	ptr = memblock_alloc_internal(size, align,
1526
+					   min_addr, max_addr, nid, true);
1527
+	if (ptr && size > 0)
1528
+		page_init_poison(ptr, size);
13841529 
13851530 	return ptr;
13861531 }
13871532 
13881533 /**
1389
- * memblock_virt_alloc_try_nid_raw - allocate boot memory block without zeroing
1534
+ * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
13901535  * memory and without panicking
13911536  * @size: size of memory block to be allocated in bytes
13921537  * @align: alignment of the region and block's size
13931538  * @min_addr: the lower bound of the memory region from where the allocation
13941539  *	  is preferred (phys address)
13951540  * @max_addr: the upper bound of the memory region from where the allocation
1396
- *	      is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1541
+ *	      is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
13971542  *	      allocate only from memory limited by memblock.current_limit value
13981543  * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
13991544  *
....@@ -1404,34 +1549,33 @@
14041549  * Return:
14051550  * Virtual address of allocated memory block on success, NULL on failure.
14061551  */
1407
-void * __init memblock_virt_alloc_try_nid_raw(
1552
+void * __init memblock_alloc_try_nid_raw(
14081553 			phys_addr_t size, phys_addr_t align,
14091554 			phys_addr_t min_addr, phys_addr_t max_addr,
14101555 			int nid)
14111556 {
14121557 	void *ptr;
14131558 
1414
-	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1559
+	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
14151560 		     __func__, (u64)size, (u64)align, nid, &min_addr,
14161561 		     &max_addr, (void *)_RET_IP_);
14171562 
1418
-	ptr = memblock_virt_alloc_internal(size, align,
1419
-					   min_addr, max_addr, nid);
1420
-#ifdef CONFIG_DEBUG_VM
1563
+	ptr = memblock_alloc_internal(size, align,
1564
+					   min_addr, max_addr, nid, false);
14211565 	if (ptr && size > 0)
1422
-		memset(ptr, PAGE_POISON_PATTERN, size);
1423
-#endif
1566
+		page_init_poison(ptr, size);
1567
+
14241568 	return ptr;
14251569 }
14261570 
14271571 /**
1428
- * memblock_virt_alloc_try_nid_nopanic - allocate boot memory block
1572
+ * memblock_alloc_try_nid - allocate boot memory block
14291573  * @size: size of memory block to be allocated in bytes
14301574  * @align: alignment of the region and block's size
14311575  * @min_addr: the lower bound of the memory region from where the allocation
14321576  *	  is preferred (phys address)
14331577  * @max_addr: the upper bound of the memory region from where the allocation
1434
- *	      is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1578
+ *	      is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
14351579  *	      allocate only from memory limited by memblock.current_limit value
14361580  * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
14371581  *
....@@ -1441,101 +1585,47 @@
14411585  * Return:
14421586  * Virtual address of allocated memory block on success, NULL on failure.
14431587  */
1444
-void * __init memblock_virt_alloc_try_nid_nopanic(
1445
-				phys_addr_t size, phys_addr_t align,
1446
-				phys_addr_t min_addr, phys_addr_t max_addr,
1447
-				int nid)
1448
-{
1449
-	void *ptr;
1450
-
1451
-	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1452
-		     __func__, (u64)size, (u64)align, nid, &min_addr,
1453
-		     &max_addr, (void *)_RET_IP_);
1454
-
1455
-	ptr = memblock_virt_alloc_internal(size, align,
1456
-					   min_addr, max_addr, nid);
1457
-	if (ptr)
1458
-		memset(ptr, 0, size);
1459
-	return ptr;
1460
-}
1461
-
1462
-/**
1463
- * memblock_virt_alloc_try_nid - allocate boot memory block with panicking
1464
- * @size: size of memory block to be allocated in bytes
1465
- * @align: alignment of the region and block's size
1466
- * @min_addr: the lower bound of the memory region from where the allocation
1467
- *	  is preferred (phys address)
1468
- * @max_addr: the upper bound of the memory region from where the allocation
1469
- *	      is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to
1470
- *	      allocate only from memory limited by memblock.current_limit value
1471
- * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1472
- *
1473
- * Public panicking version of memblock_virt_alloc_try_nid_nopanic()
1474
- * which provides debug information (including caller info), if enabled,
1475
- * and panics if the request can not be satisfied.
1476
- *
1477
- * Return:
1478
- * Virtual address of allocated memory block on success, NULL on failure.
1479
- */
1480
-void * __init memblock_virt_alloc_try_nid(
1588
+void * __init memblock_alloc_try_nid(
14811589 			phys_addr_t size, phys_addr_t align,
14821590 			phys_addr_t min_addr, phys_addr_t max_addr,
14831591 			int nid)
14841592 {
14851593 	void *ptr;
14861594 
1487
-	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n",
1595
+	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
14881596 		     __func__, (u64)size, (u64)align, nid, &min_addr,
14891597 		     &max_addr, (void *)_RET_IP_);
1490
-	ptr = memblock_virt_alloc_internal(size, align,
1491
-					   min_addr, max_addr, nid);
1492
-	if (ptr) {
1598
+	ptr = memblock_alloc_internal(size, align,
1599
+					   min_addr, max_addr, nid, false);
1600
+	if (ptr)
14931601 		memset(ptr, 0, size);
1494
-		return ptr;
1495
-	}
14961602 
1497
-	panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa\n",
1498
-	      __func__, (u64)size, (u64)align, nid, &min_addr, &max_addr);
1499
-	return NULL;
1500
-}
1501
-#endif
1502
-
1503
-/**
1504
- * __memblock_free_early - free boot memory block
1505
- * @base: phys starting address of the  boot memory block
1506
- * @size: size of the boot memory block in bytes
1507
- *
1508
- * Free boot memory block previously allocated by memblock_virt_alloc_xx() API.
1509
- * The freeing memory will not be released to the buddy allocator.
1510
- */
1511
-void __init __memblock_free_early(phys_addr_t base, phys_addr_t size)
1512
-{
1513
-	memblock_free(base, size);
1603
+	return ptr;
15141604 }
15151605 
15161606 /**
1517
- * __memblock_free_late - free bootmem block pages directly to buddy allocator
1607
+ * __memblock_free_late - free pages directly to buddy allocator
15181608  * @base: phys starting address of the  boot memory block
15191609  * @size: size of the boot memory block in bytes
15201610  *
1521
- * This is only useful when the bootmem allocator has already been torn
1611
+ * This is only useful when the memblock allocator has already been torn
15221612  * down, but we are still initializing the system.  Pages are released directly
1523
- * to the buddy allocator, no bootmem metadata is updated because it is gone.
1613
+ * to the buddy allocator.
15241614  */
15251615 void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
15261616 {
15271617 	phys_addr_t cursor, end;
15281618 
15291619 	end = base + size - 1;
1530
-	memblock_dbg("%s: [%pa-%pa] %pF\n",
1620
+	memblock_dbg("%s: [%pa-%pa] %pS\n",
15311621 		     __func__, &base, &end, (void *)_RET_IP_);
15321622 	kmemleak_free_part_phys(base, size);
15331623 	cursor = PFN_UP(base);
15341624 	end = PFN_DOWN(base + size);
15351625 
15361626 	for (; cursor < end; cursor++) {
1537
-		__free_pages_bootmem(pfn_to_page(cursor), cursor, 0);
1538
-		totalram_pages++;
1627
+		memblock_free_pages(pfn_to_page(cursor), cursor, 0);
1628
+		totalram_pages_inc();
15391629 	}
15401630 }
15411631 
....@@ -1553,23 +1643,6 @@
15531643 	return memblock.reserved.total_size;
15541644 }
15551645 
1556
-phys_addr_t __init memblock_mem_size(unsigned long limit_pfn)
1557
-{
1558
-	unsigned long pages = 0;
1559
-	struct memblock_region *r;
1560
-	unsigned long start_pfn, end_pfn;
1561
-
1562
-	for_each_memblock(memory, r) {
1563
-		start_pfn = memblock_region_memory_base_pfn(r);
1564
-		end_pfn = memblock_region_memory_end_pfn(r);
1565
-		start_pfn = min_t(unsigned long, start_pfn, limit_pfn);
1566
-		end_pfn = min_t(unsigned long, end_pfn, limit_pfn);
1567
-		pages += end_pfn - start_pfn;
1568
-	}
1569
-
1570
-	return PFN_PHYS(pages);
1571
-}
1572
-
15731646 /* lowest address */
15741647 phys_addr_t __init_memblock memblock_start_of_DRAM(void)
15751648 {
....@@ -1582,6 +1655,7 @@
15821655 
15831656 	return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
15841657 }
1658
+EXPORT_SYMBOL_GPL(memblock_end_of_DRAM);
15851659 
15861660 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
15871661 {
....@@ -1593,7 +1667,7 @@
15931667 	 * the memory memblock regions, if the @limit exceeds the total size
15941668 	 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
15951669 	 */
1596
-	for_each_memblock(memory, r) {
1670
+	for_each_mem_region(r) {
15971671 		if (limit <= r->size) {
15981672 			max_addr = r->base + limit;
15991673 			break;
....@@ -1606,7 +1680,7 @@
16061680 
16071681 void __init memblock_enforce_memory_limit(phys_addr_t limit)
16081682 {
1609
-	phys_addr_t max_addr = PHYS_ADDR_MAX;
1683
+	phys_addr_t max_addr;
16101684 
16111685 	if (!limit)
16121686 		return;
....@@ -1686,7 +1760,7 @@
16861760 	return -1;
16871761 }
16881762 
1689
-bool __init memblock_is_reserved(phys_addr_t addr)
1763
+bool __init_memblock memblock_is_reserved(phys_addr_t addr)
16901764 {
16911765 	return memblock_search(&memblock.reserved, addr) != -1;
16921766 }
....@@ -1705,7 +1779,6 @@
17051779 	return !memblock_is_nomap(&memblock.memory.regions[i]);
17061780 }
17071781 
1708
-#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
17091782 int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
17101783 			 unsigned long *start_pfn, unsigned long *end_pfn)
17111784 {
....@@ -1718,9 +1791,8 @@
17181791 	*start_pfn = PFN_DOWN(type->regions[mid].base);
17191792 	*end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
17201793 
1721
-	return type->regions[mid].nid;
1794
+	return memblock_get_region_node(&type->regions[mid]);
17221795 }
1723
-#endif
17241796 
17251797 /**
17261798  * memblock_is_region_memory - check if a region is a subset of memory
....@@ -1743,15 +1815,6 @@
17431815 		 memblock.memory.regions[idx].size) >= end;
17441816 }
17451817 
1746
-bool __init_memblock memblock_overlaps_memory(phys_addr_t base,
1747
-					      phys_addr_t size)
1748
-{
1749
-	memblock_cap_size(base, &size);
1750
-
1751
-	return memblock_overlaps_region(&memblock.memory, base, size);
1752
-}
1753
-EXPORT_SYMBOL_GPL(memblock_overlaps_memory);
1754
-
17551818 /**
17561819  * memblock_is_region_reserved - check if a region intersects reserved memory
17571820  * @base: base of region to check
....@@ -1765,7 +1828,6 @@
17651828  */
17661829 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
17671830 {
1768
-	memblock_cap_size(base, &size);
17691831 	return memblock_overlaps_region(&memblock.reserved, base, size);
17701832 }
17711833 
....@@ -1774,7 +1836,7 @@
17741836 	phys_addr_t start, end, orig_start, orig_end;
17751837 	struct memblock_region *r;
17761838 
1777
-	for_each_memblock(memory, r) {
1839
+	for_each_mem_region(r) {
17781840 		orig_start = r->base;
17791841 		orig_end = r->base + r->size;
17801842 		start = round_up(orig_start, align);
....@@ -1820,7 +1882,7 @@
18201882 		size = rgn->size;
18211883 		end = base + size - 1;
18221884 		flags = rgn->flags;
1823
-#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1885
+#ifdef CONFIG_NEED_MULTIPLE_NODES
18241886 		if (memblock_get_region_node(rgn) != MAX_NUMNODES)
18251887 			snprintf(nid_buf, sizeof(nid_buf), " on node %d",
18261888 				 memblock_get_region_node(rgn));
....@@ -1830,7 +1892,7 @@
18301892 	}
18311893 }
18321894 
1833
-void __init_memblock __memblock_dump_all(void)
1895
+static void __init_memblock __memblock_dump_all(void)
18341896 {
18351897 	pr_info("MEMBLOCK configuration:\n");
18361898 	pr_info(" memory size = %pa reserved size = %pa\n",
....@@ -1840,8 +1902,14 @@
18401902 	memblock_dump(&memblock.memory);
18411903 	memblock_dump(&memblock.reserved);
18421904 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
1843
-	memblock_dump(&memblock.physmem);
1905
+	memblock_dump(&physmem);
18441906 #endif
1907
+}
1908
+
1909
+void __init_memblock memblock_dump_all(void)
1910
+{
1911
+	if (memblock_debug)
1912
+		__memblock_dump_all();
18451913 }
18461914 
18471915 void __init memblock_allow_resize(void)
....@@ -1857,7 +1925,143 @@
18571925 }
18581926 early_param("memblock", early_memblock);
18591927 
1860
-#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_ARCH_DISCARD_MEMBLOCK)
1928
+static int __init early_memblock_nomap(char *str)
1929
+{
1930
+	return kstrtobool(str, &memblock_nomap_remove);
1931
+}
1932
+early_param("android12_only.will_be_removed_soon.memblock_nomap_remove", early_memblock_nomap);
1933
+
1934
+bool __init memblock_is_nomap_remove(void)
1935
+{
1936
+	return memblock_nomap_remove;
1937
+}
1938
+
1939
+static void __init __free_pages_memory(unsigned long start, unsigned long end)
1940
+{
1941
+	int order;
1942
+
1943
+	while (start < end) {
1944
+		order = min(MAX_ORDER - 1UL, __ffs(start));
1945
+
1946
+		while (start + (1UL << order) > end)
1947
+			order--;
1948
+
1949
+		memblock_free_pages(pfn_to_page(start), start, order);
1950
+
1951
+		start += (1UL << order);
1952
+	}
1953
+}
1954
+
1955
+#if defined(CONFIG_ROCKCHIP_THUNDER_BOOT) && defined(CONFIG_SMP)
1956
+int __init defer_free_memblock(void *unused)
1957
+{
1958
+	if (defer_start == 0)
1959
+		return 0;
1960
+
1961
+	pr_debug("start = %ld, end = %ld\n", defer_start, defer_end);
1962
+
1963
+	__free_pages_memory(defer_start, defer_end);
1964
+
1965
+	totalram_pages_add(defer_end - defer_start);
1966
+
1967
+	pr_info("%s: size %luM free %luM [%luM - %luM] total %luM\n", __func__,
1968
+		defer_free_block_size >> 20,
1969
+		(defer_end - defer_start) >> (20 - PAGE_SHIFT),
1970
+		defer_end >> (20 - PAGE_SHIFT),
1971
+		defer_start >> (20 - PAGE_SHIFT),
1972
+		totalram_pages() >> (20 - PAGE_SHIFT));
1973
+	return 0;
1974
+}
1975
+#endif
1976
+
1977
+static unsigned long __init __free_memory_core(phys_addr_t start,
1978
+				 phys_addr_t end)
1979
+{
1980
+	unsigned long start_pfn = PFN_UP(start);
1981
+	unsigned long end_pfn = min_t(unsigned long,
1982
+				      PFN_DOWN(end), max_low_pfn);
1983
+
1984
+	if (start_pfn >= end_pfn)
1985
+		return 0;
1986
+
1987
+#if defined(CONFIG_ROCKCHIP_THUNDER_BOOT) && defined(CONFIG_SMP)
1988
+	if ((end - start) > defer_free_block_size) {
1989
+		defer_start = start_pfn;
1990
+		defer_end = end_pfn;
1991
+
1992
+		return 0;
1993
+	}
1994
+#endif
1995
+
1996
+	__free_pages_memory(start_pfn, end_pfn);
1997
+
1998
+	return end_pfn - start_pfn;
1999
+}
2000
+
2001
+static unsigned long __init free_low_memory_core_early(void)
2002
+{
2003
+	unsigned long count = 0;
2004
+	phys_addr_t start, end;
2005
+	u64 i;
2006
+
2007
+	memblock_clear_hotplug(0, -1);
2008
+
2009
+	for_each_reserved_mem_range(i, &start, &end)
2010
+		reserve_bootmem_region(start, end);
2011
+
2012
+	/*
2013
+	 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
2014
+	 *  because in some case like Node0 doesn't have RAM installed
2015
+	 *  low ram will be on Node1
2016
+	 */
2017
+	for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
2018
+				NULL)
2019
+		count += __free_memory_core(start, end);
2020
+
2021
+	return count;
2022
+}
2023
+
2024
+static int reset_managed_pages_done __initdata;
2025
+
2026
+void reset_node_managed_pages(pg_data_t *pgdat)
2027
+{
2028
+	struct zone *z;
2029
+
2030
+	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
2031
+		atomic_long_set(&z->managed_pages, 0);
2032
+}
2033
+
2034
+void __init reset_all_zones_managed_pages(void)
2035
+{
2036
+	struct pglist_data *pgdat;
2037
+
2038
+	if (reset_managed_pages_done)
2039
+		return;
2040
+
2041
+	for_each_online_pgdat(pgdat)
2042
+		reset_node_managed_pages(pgdat);
2043
+
2044
+	reset_managed_pages_done = 1;
2045
+}
2046
+
2047
+/**
2048
+ * memblock_free_all - release free pages to the buddy allocator
2049
+ *
2050
+ * Return: the number of pages actually released.
2051
+ */
2052
+unsigned long __init memblock_free_all(void)
2053
+{
2054
+	unsigned long pages;
2055
+
2056
+	reset_all_zones_managed_pages();
2057
+
2058
+	pages = free_low_memory_core_early();
2059
+	totalram_pages_add(pages);
2060
+
2061
+	return pages;
2062
+}
2063
+
2064
+#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
18612065 
18622066 static int memblock_debug_show(struct seq_file *m, void *private)
18632067 {
....@@ -1880,15 +2084,14 @@
18802084 static int __init memblock_init_debugfs(void)
18812085 {
18822086 	struct dentry *root = debugfs_create_dir("memblock", NULL);
1883
-	if (!root)
1884
-		return -ENXIO;
2087
+
18852088 	debugfs_create_file("memory", 0444, root,
18862089 			    &memblock.memory, &memblock_debug_fops);
18872090 	debugfs_create_file("reserved", 0444, root,
18882091 			    &memblock.reserved, &memblock_debug_fops);
18892092 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
1890
-	debugfs_create_file("physmem", 0444, root,
1891
-			    &memblock.physmem, &memblock_debug_fops);
2093
+	debugfs_create_file("physmem", 0444, root, &physmem,
2094
+			    &memblock_debug_fops);
18922095 #endif
18932096 
18942097 	return 0;