forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc 
 kernel/arch/arm/mm/init.c
....@@ -1,17 +1,13 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *  linux/arch/arm/mm/init.c
34  *
45  *  Copyright (C) 1995-2005 Russell King
5
- *
6
- * This program is free software; you can redistribute it and/or modify
7
- * it under the terms of the GNU General Public License version 2 as
8
- * published by the Free Software Foundation.
96  */
107 #include <linux/kernel.h>
118 #include <linux/errno.h>
129 #include <linux/swap.h>
1310 #include <linux/init.h>
14
-#include <linux/bootmem.h>
1511 #include <linux/mman.h>
1612 #include <linux/sched/signal.h>
1713 #include <linux/sched/task.h>
....@@ -22,10 +18,11 @@
2218 #include <linux/highmem.h>
2319 #include <linux/gfp.h>
2420 #include <linux/memblock.h>
25
-#include <linux/dma-contiguous.h>
21
+#include <linux/dma-map-ops.h>
2622 #include <linux/sizes.h>
2723 #include <linux/stop_machine.h>
2824 #include <linux/swiotlb.h>
25
+#include <linux/rk-dma-heap.h>
2926 
3027 #include <asm/cp15.h>
3128 #include <asm/mach-types.h>
....@@ -34,6 +31,7 @@
3431 #include <asm/prom.h>
3532 #include <asm/sections.h>
3633 #include <asm/setup.h>
34
+#include <asm/set_memory.h>
3735 #include <asm/system_info.h>
3836 #include <asm/tlb.h>
3937 #include <asm/fixmap.h>
....@@ -52,26 +50,7 @@
5250 }
5351 #endif
5452 
55
-static phys_addr_t phys_initrd_start __initdata = 0;
56
-static unsigned long phys_initrd_size __initdata = 0;
57
-
58
-static int __init early_initrd(char *p)
59
-{
60
-	phys_addr_t start;
61
-	unsigned long size;
62
-	char *endp;
63
-
64
-	start = memparse(p, &endp);
65
-	if (*endp == ',') {
66
-		size = memparse(endp + 1, NULL);
67
-
68
-		phys_initrd_start = start;
69
-		phys_initrd_size = size;
70
-	}
71
-	return 0;
72
-}
73
-early_param("initrd", early_initrd);
74
-
53
+#ifdef CONFIG_BLK_DEV_INITRD
7554 static int __init parse_tag_initrd(const struct tag *tag)
7655 {
7756 	pr_warn("ATAG_INITRD is deprecated; "
....@@ -91,6 +70,7 @@
9170 }
9271 
9372 __tagtable(ATAG_INITRD2, parse_tag_initrd2);
73
+#endif
9474 
9575 static void __init find_limits(unsigned long *min, unsigned long *max_low,
9676 			       unsigned long *max_high)
....@@ -113,18 +93,6 @@
11393  */
11494 phys_addr_t arm_dma_limit;
11595 unsigned long arm_dma_pfn_limit;
116
-
117
-static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
118
-	unsigned long dma_size)
119
-{
120
-	if (size[0] <= dma_size)
121
-		return;
122
-
123
-	size[ZONE_NORMAL] = size[0] - dma_size;
124
-	size[ZONE_DMA] = dma_size;
125
-	hole[ZONE_NORMAL] = hole[0];
126
-	hole[ZONE_DMA] = 0;
127
-}
12896 #endif
12997 
13098 void __init setup_dma_zone(const struct machine_desc *mdesc)
....@@ -142,84 +110,40 @@
142110 static void __init zone_sizes_init(unsigned long min, unsigned long max_low,
143111 	unsigned long max_high)
144112 {
145
-	unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
146
-	struct memblock_region *reg;
147
-
148
-	/*
149
-	 * initialise the zones.
150
-	 */
151
-	memset(zone_size, 0, sizeof(zone_size));
152
-
153
-	/*
154
-	 * The memory size has already been determined.  If we need
155
-	 * to do anything fancy with the allocation of this memory
156
-	 * to the zones, now is the time to do it.
157
-	 */
158
-	zone_size[0] = max_low - min;
159
-#ifdef CONFIG_HIGHMEM
160
-	zone_size[ZONE_HIGHMEM] = max_high - max_low;
161
-#endif
162
-
163
-	/*
164
-	 * Calculate the size of the holes.
165
-	 *  holes = node_size - sum(bank_sizes)
166
-	 */
167
-	memcpy(zhole_size, zone_size, sizeof(zhole_size));
168
-	for_each_memblock(memory, reg) {
169
-		unsigned long start = memblock_region_memory_base_pfn(reg);
170
-		unsigned long end = memblock_region_memory_end_pfn(reg);
171
-
172
-		if (start < max_low) {
173
-			unsigned long low_end = min(end, max_low);
174
-			zhole_size[0] -= low_end - start;
175
-		}
176
-#ifdef CONFIG_HIGHMEM
177
-		if (end > max_low) {
178
-			unsigned long high_start = max(start, max_low);
179
-			zhole_size[ZONE_HIGHMEM] -= end - high_start;
180
-		}
181
-#endif
182
-	}
113
+	unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
183114 
184115 #ifdef CONFIG_ZONE_DMA
185
-	/*
186
-	 * Adjust the sizes according to any special requirements for
187
-	 * this machine type.
188
-	 */
189
-	if (arm_dma_zone_size)
190
-		arm_adjust_dma_zone(zone_size, zhole_size,
191
-			arm_dma_zone_size >> PAGE_SHIFT);
116
+	max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low);
192117 #endif
193
-
194
-	free_area_init_node(0, zone_size, min, zhole_size);
118
+	max_zone_pfn[ZONE_NORMAL] = max_low;
119
+#ifdef CONFIG_HIGHMEM
120
+	max_zone_pfn[ZONE_HIGHMEM] = max_high;
121
+#endif
122
+	free_area_init(max_zone_pfn);
195123 }
196124 
197125 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
198126 int pfn_valid(unsigned long pfn)
199127 {
200128 	phys_addr_t addr = __pfn_to_phys(pfn);
129
+	unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages;
201130 
202131 	if (__phys_to_pfn(addr) != pfn)
203132 		return 0;
204133 
205
-	return memblock_is_map_memory(__pfn_to_phys(pfn));
134
+	/*
135
+	 * If address less than pageblock_size bytes away from a present
136
+	 * memory chunk there still will be a memory map entry for it
137
+	 * because we round freed memory map to the pageblock boundaries.
138
+	 */
139
+	if (memblock_overlaps_region(&memblock.memory,
140
+				     ALIGN_DOWN(addr, pageblock_size),
141
+				     pageblock_size))
142
+		return 1;
143
+
144
+	return 0;
206145 }
207146 EXPORT_SYMBOL(pfn_valid);
208
-#endif
209
-
210
-#ifndef CONFIG_SPARSEMEM
211
-static void __init arm_memory_present(void)
212
-{
213
-}
214
-#else
215
-static void __init arm_memory_present(void)
216
-{
217
-	struct memblock_region *reg;
218
-
219
-	for_each_memblock(memory, reg)
220
-		memory_present(0, memblock_region_memory_base_pfn(reg),
221
-			       memblock_region_memory_end_pfn(reg));
222
-}
223147 #endif
224148 
225149 static bool arm_memblock_steal_permitted = true;
....@@ -230,7 +154,11 @@
230154 
231155 	BUG_ON(!arm_memblock_steal_permitted);
232156 
233
-	phys = memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
157
+	phys = memblock_phys_alloc(size, align);
158
+	if (!phys)
159
+		panic("Failed to steal %pa bytes at %pS\n",
160
+		      &size, (void *)_RET_IP_);
161
+
234162 	memblock_free(phys, size);
235163 	memblock_remove(phys, size);
236164 
....@@ -242,12 +170,6 @@
242170 #ifdef CONFIG_BLK_DEV_INITRD
243171 	phys_addr_t start;
244172 	unsigned long size;
245
-
246
-	/* FDT scan will populate initrd_start */
247
-	if (initrd_start && !phys_initrd_size) {
248
-		phys_initrd_start = __virt_to_phys(initrd_start);
249
-		phys_initrd_size = initrd_end - initrd_start;
250
-	}
251173 
252174 	initrd_start = initrd_end = 0;
253175 
....@@ -284,6 +206,22 @@
284206 #endif
285207 }
286208 
209
+#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND
210
+void check_cpu_icache_size(int cpuid)
211
+{
212
+	u32 size, ctr;
213
+
214
+	asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr));
215
+
216
+	size = 1 << ((ctr & 0xf) + 2);
217
+	if (cpuid != 0 && icache_size != size)
218
+		pr_info("CPU%u: detected I-Cache line size mismatch, workaround enabled\n",
219
+			cpuid);
220
+	if (icache_size > size)
221
+		icache_size = size;
222
+}
223
+#endif
224
+
287225 void __init arm_memblock_init(const struct machine_desc *mdesc)
288226 {
289227 	/* Register the kernel text, kernel data and initrd with memblock. */
....@@ -297,11 +235,11 @@
297235 	if (mdesc->reserve)
298236 		mdesc->reserve();
299237 
300
-	early_init_fdt_reserve_self();
301238 	early_init_fdt_scan_reserved_mem();
302239 
303240 	/* reserve memory for DMA contiguous allocations */
304241 	dma_contiguous_reserve(arm_dma_limit);
242
+	rk_dma_heap_cma_setup();
305243 
306244 	arm_memblock_steal_permitted = false;
307245 	memblock_dump_all();
....@@ -309,42 +247,25 @@
309247 
310248 void __init bootmem_init(void)
311249 {
312
-	unsigned long min, max_low, max_high;
313
-
314250 	memblock_allow_resize();
315
-	max_low = max_high = 0;
316251 
317
-	find_limits(&min, &max_low, &max_high);
252
+	find_limits(&min_low_pfn, &max_low_pfn, &max_pfn);
318253 
319
-	early_memtest((phys_addr_t)min << PAGE_SHIFT,
320
-		      (phys_addr_t)max_low << PAGE_SHIFT);
254
+	early_memtest((phys_addr_t)min_low_pfn << PAGE_SHIFT,
255
+		      (phys_addr_t)max_low_pfn << PAGE_SHIFT);
321256 
322257 	/*
323
-	 * Sparsemem tries to allocate bootmem in memory_present(),
324
-	 * so must be done after the fixed reservations
325
-	 */
326
-	arm_memory_present();
327
-
328
-	/*
329
-	 * sparse_init() needs the bootmem allocator up and running.
258
+	 * sparse_init() tries to allocate memory from memblock, so must be
259
+	 * done after the fixed reservations
330260 	 */
331261 	sparse_init();
332262 
333263 	/*
334
-	 * Now free the memory - free_area_init_node needs
264
+	 * Now free the memory - free_area_init needs
335265 	 * the sparse mem_map arrays initialized by sparse_init()
336266 	 * for memmap_init_zone(), otherwise all PFNs are invalid.
337267 	 */
338
-	zone_sizes_init(min, max_low, max_high);
339
-
340
-	/*
341
-	 * This doesn't seem to be used by the Linux memory manager any
342
-	 * more, but is used by ll_rw_block.  If we can get rid of it, we
343
-	 * also get rid of some of the stuff above as well.
344
-	 */
345
-	min_low_pfn = min;
346
-	max_low_pfn = max_low;
347
-	max_pfn = max_high;
268
+	zone_sizes_init(min_low_pfn, max_low_pfn, max_pfn);
348269 }
349270 
350271 /*
....@@ -390,16 +311,14 @@
390311  */
391312 static void __init free_unused_memmap(void)
392313 {
393
-	unsigned long start, prev_end = 0;
394
-	struct memblock_region *reg;
314
+	unsigned long start, end, prev_end = 0;
315
+	int i;
395316 
396317 	/*
397318 	 * This relies on each bank being in address order.
398319 	 * The banks are sorted previously in bootmem_init().
399320 	 */
400
-	for_each_memblock(memory, reg) {
401
-		start = memblock_region_memory_base_pfn(reg);
402
-
321
+	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {
403322 #ifdef CONFIG_SPARSEMEM
404323 		/*
405324 		 * Take care not to free memmap entries that don't exist
....@@ -407,14 +326,14 @@
407326 		 */
408327 		start = min(start,
409328 				 ALIGN(prev_end, PAGES_PER_SECTION));
410
-#else
411
-		/*
412
-		 * Align down here since the VM subsystem insists that the
413
-		 * memmap entries are valid from the bank start aligned to
414
-		 * MAX_ORDER_NR_PAGES.
415
-		 */
416
-		start = round_down(start, MAX_ORDER_NR_PAGES);
417329 #endif
330
+		/*
331
+		 * Align down here since many operations in VM subsystem
332
+		 * presume that there are no holes in the memory map inside
333
+		 * a pageblock
334
+		 */
335
+		start = round_down(start, pageblock_nr_pages);
336
+
418337 		/*
419338 		 * If we had a previous bank, and there is a space
420339 		 * between the current bank and the previous, free it.
....@@ -423,76 +342,45 @@
423342 			free_memmap(prev_end, start);
424343 
425344 		/*
426
-		 * Align up here since the VM subsystem insists that the
427
-		 * memmap entries are valid from the bank end aligned to
428
-		 * MAX_ORDER_NR_PAGES.
345
+		 * Align up here since many operations in VM subsystem
346
+		 * presume that there are no holes in the memory map inside
347
+		 * a pageblock
429348 		 */
430
-		prev_end = ALIGN(memblock_region_memory_end_pfn(reg),
431
-				 MAX_ORDER_NR_PAGES);
349
+		prev_end = ALIGN(end, pageblock_nr_pages);
432350 	}
433351 
434352 #ifdef CONFIG_SPARSEMEM
435
-	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
353
+	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
354
+		prev_end = ALIGN(end, pageblock_nr_pages);
436355 		free_memmap(prev_end,
437356 			    ALIGN(prev_end, PAGES_PER_SECTION));
357
+	}
438358 #endif
439359 }
440
-
441
-#ifdef CONFIG_HIGHMEM
442
-static inline void free_area_high(unsigned long pfn, unsigned long end)
443
-{
444
-	for (; pfn < end; pfn++)
445
-		free_highmem_page(pfn_to_page(pfn));
446
-}
447
-#endif
448360 
449361 static void __init free_highpages(void)
450362 {
451363 #ifdef CONFIG_HIGHMEM
452364 	unsigned long max_low = max_low_pfn;
453
-	struct memblock_region *mem, *res;
365
+	phys_addr_t range_start, range_end;
366
+	u64 i;
454367 
455368 	/* set highmem page free */
456
-	for_each_memblock(memory, mem) {
457
-		unsigned long start = memblock_region_memory_base_pfn(mem);
458
-		unsigned long end = memblock_region_memory_end_pfn(mem);
369
+	for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
370
+				&range_start, &range_end, NULL) {
371
+		unsigned long start = PFN_UP(range_start);
372
+		unsigned long end = PFN_DOWN(range_end);
459373 
460374 		/* Ignore complete lowmem entries */
461375 		if (end <= max_low)
462
-			continue;
463
-
464
-		if (memblock_is_nomap(mem))
465376 			continue;
466377 
467378 		/* Truncate partial highmem entries */
468379 		if (start < max_low)
469380 			start = max_low;
470381 
471
-		/* Find and exclude any reserved regions */
472
-		for_each_memblock(reserved, res) {
473
-			unsigned long res_start, res_end;
474
-
475
-			res_start = memblock_region_reserved_base_pfn(res);
476
-			res_end = memblock_region_reserved_end_pfn(res);
477
-
478
-			if (res_end < start)
479
-				continue;
480
-			if (res_start < start)
481
-				res_start = start;
482
-			if (res_start > end)
483
-				res_start = end;
484
-			if (res_end > end)
485
-				res_end = end;
486
-			if (res_start != start)
487
-				free_area_high(start, res_start);
488
-			start = res_end;
489
-			if (start == end)
490
-				break;
491
-		}
492
-
493
-		/* And now free anything which remains */
494
-		if (start < end)
495
-			free_area_high(start, end);
382
+		for (; start < end; start++)
383
+			free_highmem_page(pfn_to_page(start));
496384 	}
497385 #endif
498386 }
....@@ -504,21 +392,19 @@
504392  */
505393 void __init mem_init(void)
506394 {
507
-#ifdef CONFIG_HAVE_TCM
508
-	/* These pointers are filled in on TCM detection */
509
-	extern u32 dtcm_end;
510
-	extern u32 itcm_end;
511
-#endif
512
-
513395 #ifdef CONFIG_ARM_LPAE
514
-	swiotlb_init(1);
396
+	if (swiotlb_force == SWIOTLB_FORCE ||
397
+	    max_pfn > arm_dma_pfn_limit)
398
+		swiotlb_init(1);
399
+	else
400
+		swiotlb_force = SWIOTLB_NO_FORCE;
515401 #endif
516402 
517403 	set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
518404 
519405 	/* this will put all unused low memory onto the freelists */
520406 	free_unused_memmap();
521
-	free_all_bootmem();
407
+	memblock_free_all();
522408 
523409 #ifdef CONFIG_SA1111
524410 	/* now that our DMA memory is actually so designated, we can free it */
....@@ -528,55 +414,6 @@
528414 	free_highpages();
529415 
530416 	mem_init_print_info(NULL);
531
-
532
-#define MLK(b, t) b, t, ((t) - (b)) >> 10
533
-#define MLM(b, t) b, t, ((t) - (b)) >> 20
534
-#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
535
-
536
-	pr_notice("Virtual kernel memory layout:\n"
537
-			"    vector  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
538
-#ifdef CONFIG_HAVE_TCM
539
-			"    DTCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
540
-			"    ITCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
541
-#endif
542
-			"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
543
-			"    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
544
-			"    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
545
-#ifdef CONFIG_HIGHMEM
546
-			"    pkmap   : 0x%08lx - 0x%08lx   (%4ld MB)\n"
547
-#endif
548
-#ifdef CONFIG_MODULES
549
-			"    modules : 0x%08lx - 0x%08lx   (%4ld MB)\n"
550
-#endif
551
-			"      .text : 0x%px" " - 0x%px" "   (%4td kB)\n"
552
-			"      .init : 0x%px" " - 0x%px" "   (%4td kB)\n"
553
-			"      .data : 0x%px" " - 0x%px" "   (%4td kB)\n"
554
-			"       .bss : 0x%px" " - 0x%px" "   (%4td kB)\n",
555
-
556
-			MLK(VECTORS_BASE, VECTORS_BASE + PAGE_SIZE),
557
-#ifdef CONFIG_HAVE_TCM
558
-			MLK(DTCM_OFFSET, (unsigned long) dtcm_end),
559
-			MLK(ITCM_OFFSET, (unsigned long) itcm_end),
560
-#endif
561
-			MLK(FIXADDR_START, FIXADDR_END),
562
-			MLM(VMALLOC_START, VMALLOC_END),
563
-			MLM(PAGE_OFFSET, (unsigned long)high_memory),
564
-#ifdef CONFIG_HIGHMEM
565
-			MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
566
-				(PAGE_SIZE)),
567
-#endif
568
-#ifdef CONFIG_MODULES
569
-			MLM(MODULES_VADDR, MODULES_END),
570
-#endif
571
-
572
-			MLK_ROUNDUP(_text, _etext),
573
-			MLK_ROUNDUP(__init_begin, __init_end),
574
-			MLK_ROUNDUP(_sdata, _edata),
575
-			MLK_ROUNDUP(__bss_start, __bss_stop));
576
-
577
-#undef MLK
578
-#undef MLM
579
-#undef MLK_ROUNDUP
580417 
581418 	/*
582419 	 * Check boundaries twice: Some fundamental inconsistencies can
....@@ -660,7 +497,7 @@
660497 {
661498 	pmd_t *pmd;
662499 
663
-	pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
500
+	pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr);
664501 
665502 #ifdef CONFIG_ARM_LPAE
666503 	pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
....@@ -683,8 +520,8 @@
683520 	return !!(get_cr() & CR_XP);
684521 }
685522 
686
-void set_section_perms(struct section_perm *perms, int n, bool set,
687
-			struct mm_struct *mm)
523
+static void set_section_perms(struct section_perm *perms, int n, bool set,
524
+			      struct mm_struct *mm)
688525 {
689526 	size_t i;
690527 	unsigned long addr;
....@@ -788,27 +625,14 @@
788625 }
789626 
790627 #ifdef CONFIG_BLK_DEV_INITRD
791
-
792
-static int keep_initrd;
793
-
794628 void free_initrd_mem(unsigned long start, unsigned long end)
795629 {
796
-	if (!keep_initrd) {
797
-		if (start == initrd_start)
798
-			start = round_down(start, PAGE_SIZE);
799
-		if (end == initrd_end)
800
-			end = round_up(end, PAGE_SIZE);
630
+	if (start == initrd_start)
631
+		start = round_down(start, PAGE_SIZE);
632
+	if (end == initrd_end)
633
+		end = round_up(end, PAGE_SIZE);
801634 
802
-		poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
803
-		free_reserved_area((void *)start, (void *)end, -1, "initrd");
804
-	}
635
+	poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
636
+	free_reserved_area((void *)start, (void *)end, -1, "initrd");
805637 }
806
-
807
-static int __init keepinitrd_setup(char *__unused)
808
-{
809
-	keep_initrd = 1;
810
-	return 1;
811
-}
812
-
813
-__setup("keepinitrd", keepinitrd_setup);
814638 #endif