add new system file

hc

2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c

 kernel/arch/riscv/mm/init.c
..
..
@@ -1,53 +1,96 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * Copyright (C) 2012 Regents of the University of California
3
- *
4
- *   This program is free software; you can redistribute it and/or
5
- *   modify it under the terms of the GNU General Public License
6
- *   as published by the Free Software Foundation, version 2.
7
- *
8
- *   This program is distributed in the hope that it will be useful,
9
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
10
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
- *   GNU General Public License for more details.
4
+ * Copyright (C) 2019 Western Digital Corporation or its affiliates.
12
5
  */
13
6
 
14
7
 #include <linux/init.h>
15
8
 #include <linux/mm.h>
16
-#include <linux/bootmem.h>
17
-#include <linux/initrd.h>
18
9
 #include <linux/memblock.h>
10
+#include <linux/initrd.h>
19
11
 #include <linux/swap.h>
20
12
 #include <linux/sizes.h>
13
+#include <linux/of_fdt.h>
14
+#include <linux/libfdt.h>
15
+#include <linux/set_memory.h>
16
+#include <linux/dma-map-ops.h>
21
17
 
18
+#include <asm/fixmap.h>
22
19
 #include <asm/tlbflush.h>
23
20
 #include <asm/sections.h>
24
-#include <asm/pgtable.h>
21
+#include <asm/soc.h>
25
22
 #include <asm/io.h>
23
+#include <asm/ptdump.h>
24
+
25
+#include "../kernel/head.h"
26
+
27
+unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
28
+							__page_aligned_bss;
29
+EXPORT_SYMBOL(empty_zero_page);
30
+
31
+extern char _start[];
32
+#define DTB_EARLY_BASE_VA      PGDIR_SIZE
33
+void *dtb_early_va __initdata;
34
+uintptr_t dtb_early_pa __initdata;
35
+
36
+struct pt_alloc_ops {
37
+	pte_t *(*get_pte_virt)(phys_addr_t pa);
38
+	phys_addr_t (*alloc_pte)(uintptr_t va);
39
+#ifndef __PAGETABLE_PMD_FOLDED
40
+	pmd_t *(*get_pmd_virt)(phys_addr_t pa);
41
+	phys_addr_t (*alloc_pmd)(uintptr_t va);
42
+#endif
43
+};
44
+
45
+static phys_addr_t dma32_phys_limit __ro_after_init;
26
46
 
27
47
 static void __init zone_sizes_init(void)
28
48
 {
29
49
 	unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };
30
50
 
31
51
 #ifdef CONFIG_ZONE_DMA32
32
-	max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G,
33
-			(unsigned long) PFN_PHYS(max_low_pfn)));
52
+	max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit);
34
53
 #endif
35
54
 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
36
55
 
37
-	free_area_init_nodes(max_zone_pfns);
56
+	free_area_init(max_zone_pfns);
38
57
 }
39
58
 
40
-void setup_zero_page(void)
59
+static void setup_zero_page(void)
41
60
 {
42
61
 	memset((void *)empty_zero_page, 0, PAGE_SIZE);
43
62
 }
44
63
 
45
-void __init paging_init(void)
64
+#if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM)
65
+static inline void print_mlk(char *name, unsigned long b, unsigned long t)
46
66
 {
47
-	setup_zero_page();
48
-	local_flush_tlb_all();
49
-	zone_sizes_init();
67
+	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld kB)\n", name, b, t,
68
+		  (((t) - (b)) >> 10));
50
69
 }
70
+
71
+static inline void print_mlm(char *name, unsigned long b, unsigned long t)
72
+{
73
+	pr_notice("%12s : 0x%08lx - 0x%08lx   (%4ld MB)\n", name, b, t,
74
+		  (((t) - (b)) >> 20));
75
+}
76
+
77
+static void print_vm_layout(void)
78
+{
79
+	pr_notice("Virtual kernel memory layout:\n");
80
+	print_mlk("fixmap", (unsigned long)FIXADDR_START,
81
+		  (unsigned long)FIXADDR_TOP);
82
+	print_mlm("pci io", (unsigned long)PCI_IO_START,
83
+		  (unsigned long)PCI_IO_END);
84
+	print_mlm("vmemmap", (unsigned long)VMEMMAP_START,
85
+		  (unsigned long)VMEMMAP_END);
86
+	print_mlm("vmalloc", (unsigned long)VMALLOC_START,
87
+		  (unsigned long)VMALLOC_END);
88
+	print_mlm("lowmem", (unsigned long)PAGE_OFFSET,
89
+		  (unsigned long)high_memory);
90
+}
91
+#else
92
+static void print_vm_layout(void) { }
93
+#endif /* CONFIG_DEBUG_VM */
51
94
 
52
95
 void __init mem_init(void)
53
96
 {
..
..
@@ -56,18 +99,589 @@
56
99
 #endif /* CONFIG_FLATMEM */
57
100
 
58
101
 	high_memory = (void *)(__va(PFN_PHYS(max_low_pfn)));
59
-	free_all_bootmem();
102
+	memblock_free_all();
60
103
 
61
104
 	mem_init_print_info(NULL);
62
-}
63
-
64
-void free_initmem(void)
65
-{
66
-	free_initmem_default(0);
105
+	print_vm_layout();
67
106
 }
68
107
 
69
108
 #ifdef CONFIG_BLK_DEV_INITRD
70
-void free_initrd_mem(unsigned long start, unsigned long end)
109
+static void __init setup_initrd(void)
71
110
 {
111
+	phys_addr_t start;
112
+	unsigned long size;
113
+
114
+	/* Ignore the virtul address computed during device tree parsing */
115
+	initrd_start = initrd_end = 0;
116
+
117
+	if (!phys_initrd_size)
118
+		return;
119
+	/*
120
+	 * Round the memory region to page boundaries as per free_initrd_mem()
121
+	 * This allows us to detect whether the pages overlapping the initrd
122
+	 * are in use, but more importantly, reserves the entire set of pages
123
+	 * as we don't want these pages allocated for other purposes.
124
+	 */
125
+	start = round_down(phys_initrd_start, PAGE_SIZE);
126
+	size = phys_initrd_size + (phys_initrd_start - start);
127
+	size = round_up(size, PAGE_SIZE);
128
+
129
+	if (!memblock_is_region_memory(start, size)) {
130
+		pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region",
131
+		       (u64)start, size);
132
+		goto disable;
133
+	}
134
+
135
+	if (memblock_is_region_reserved(start, size)) {
136
+		pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n",
137
+		       (u64)start, size);
138
+		goto disable;
139
+	}
140
+
141
+	memblock_reserve(start, size);
142
+	/* Now convert initrd to virtual addresses */
143
+	initrd_start = (unsigned long)__va(phys_initrd_start);
144
+	initrd_end = initrd_start + phys_initrd_size;
145
+	initrd_below_start_ok = 1;
146
+
147
+	pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n",
148
+		(void *)(initrd_start), size);
149
+	return;
150
+disable:
151
+	pr_cont(" - disabling initrd\n");
152
+	initrd_start = 0;
153
+	initrd_end = 0;
72
154
 }
73
155
 #endif /* CONFIG_BLK_DEV_INITRD */
156
+
157
+void __init setup_bootmem(void)
158
+{
159
+	phys_addr_t mem_start = 0;
160
+	phys_addr_t start, dram_end, end = 0;
161
+	phys_addr_t vmlinux_end = __pa_symbol(&_end);
162
+	phys_addr_t vmlinux_start = __pa_symbol(&_start);
163
+	phys_addr_t max_mapped_addr = __pa(~(ulong)0);
164
+	u64 i;
165
+
166
+	/* Find the memory region containing the kernel */
167
+	for_each_mem_range(i, &start, &end) {
168
+		phys_addr_t size = end - start;
169
+		if (!mem_start)
170
+			mem_start = start;
171
+		if (start <= vmlinux_start && vmlinux_end <= end)
172
+			BUG_ON(size == 0);
173
+	}
174
+
175
+	/*
176
+	 * The maximal physical memory size is -PAGE_OFFSET.
177
+	 * Make sure that any memory beyond mem_start + (-PAGE_OFFSET) is removed
178
+	 * as it is unusable by kernel.
179
+	 */
180
+	memblock_enforce_memory_limit(-PAGE_OFFSET);
181
+
182
+	/* Reserve from the start of the kernel to the end of the kernel */
183
+	memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start);
184
+
185
+	dram_end = memblock_end_of_DRAM();
186
+
187
+	/*
188
+	 * memblock allocator is not aware of the fact that last 4K bytes of
189
+	 * the addressable memory can not be mapped because of IS_ERR_VALUE
190
+	 * macro. Make sure that last 4k bytes are not usable by memblock
191
+	 * if end of dram is equal to maximum addressable memory.
192
+	 */
193
+	if (max_mapped_addr == (dram_end - 1))
194
+		memblock_set_current_limit(max_mapped_addr - 4096);
195
+
196
+	max_pfn = PFN_DOWN(dram_end);
197
+	max_low_pfn = max_pfn;
198
+	dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn));
199
+	set_max_mapnr(max_low_pfn);
200
+
201
+#ifdef CONFIG_BLK_DEV_INITRD
202
+	setup_initrd();
203
+#endif /* CONFIG_BLK_DEV_INITRD */
204
+
205
+	/*
206
+	 * Avoid using early_init_fdt_reserve_self() since __pa() does
207
+	 * not work for DTB pointers that are fixmap addresses
208
+	 */
209
+	memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va));
210
+
211
+	dma_contiguous_reserve(dma32_phys_limit);
212
+	memblock_allow_resize();
213
+	memblock_dump_all();
214
+}
215
+
216
+#ifdef CONFIG_MMU
217
+static struct pt_alloc_ops pt_ops;
218
+
219
+unsigned long va_pa_offset;
220
+EXPORT_SYMBOL(va_pa_offset);
221
+unsigned long pfn_base;
222
+EXPORT_SYMBOL(pfn_base);
223
+
224
+pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
225
+pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
226
+pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;
227
+
228
+pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
229
+
230
+void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
231
+{
232
+	unsigned long addr = __fix_to_virt(idx);
233
+	pte_t *ptep;
234
+
235
+	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
236
+
237
+	ptep = &fixmap_pte[pte_index(addr)];
238
+
239
+	if (pgprot_val(prot))
240
+		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot));
241
+	else
242
+		pte_clear(&init_mm, addr, ptep);
243
+	local_flush_tlb_page(addr);
244
+}
245
+
246
+static inline pte_t *__init get_pte_virt_early(phys_addr_t pa)
247
+{
248
+	return (pte_t *)((uintptr_t)pa);
249
+}
250
+
251
+static inline pte_t *__init get_pte_virt_fixmap(phys_addr_t pa)
252
+{
253
+	clear_fixmap(FIX_PTE);
254
+	return (pte_t *)set_fixmap_offset(FIX_PTE, pa);
255
+}
256
+
257
+static inline pte_t *get_pte_virt_late(phys_addr_t pa)
258
+{
259
+	return (pte_t *) __va(pa);
260
+}
261
+
262
+static inline phys_addr_t __init alloc_pte_early(uintptr_t va)
263
+{
264
+	/*
265
+	 * We only create PMD or PGD early mappings so we
266
+	 * should never reach here with MMU disabled.
267
+	 */
268
+	BUG();
269
+}
270
+
271
+static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va)
272
+{
273
+	return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
274
+}
275
+
276
+static phys_addr_t alloc_pte_late(uintptr_t va)
277
+{
278
+	unsigned long vaddr;
279
+
280
+	vaddr = __get_free_page(GFP_KERNEL);
281
+	if (!vaddr || !pgtable_pte_page_ctor(virt_to_page(vaddr)))
282
+		BUG();
283
+	return __pa(vaddr);
284
+}
285
+
286
+static void __init create_pte_mapping(pte_t *ptep,
287
+				      uintptr_t va, phys_addr_t pa,
288
+				      phys_addr_t sz, pgprot_t prot)
289
+{
290
+	uintptr_t pte_idx = pte_index(va);
291
+
292
+	BUG_ON(sz != PAGE_SIZE);
293
+
294
+	if (pte_none(ptep[pte_idx]))
295
+		ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot);
296
+}
297
+
298
+#ifndef __PAGETABLE_PMD_FOLDED
299
+
300
+pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss;
301
+pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss;
302
+pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
303
+pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
304
+
305
+static pmd_t *__init get_pmd_virt_early(phys_addr_t pa)
306
+{
307
+	/* Before MMU is enabled */
308
+	return (pmd_t *)((uintptr_t)pa);
309
+}
310
+
311
+static pmd_t *__init get_pmd_virt_fixmap(phys_addr_t pa)
312
+{
313
+	clear_fixmap(FIX_PMD);
314
+	return (pmd_t *)set_fixmap_offset(FIX_PMD, pa);
315
+}
316
+
317
+static pmd_t *get_pmd_virt_late(phys_addr_t pa)
318
+{
319
+	return (pmd_t *) __va(pa);
320
+}
321
+
322
+static phys_addr_t __init alloc_pmd_early(uintptr_t va)
323
+{
324
+	BUG_ON((va - PAGE_OFFSET) >> PGDIR_SHIFT);
325
+
326
+	return (uintptr_t)early_pmd;
327
+}
328
+
329
+static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va)
330
+{
331
+	return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
332
+}
333
+
334
+static phys_addr_t alloc_pmd_late(uintptr_t va)
335
+{
336
+	unsigned long vaddr;
337
+
338
+	vaddr = __get_free_page(GFP_KERNEL);
339
+	BUG_ON(!vaddr);
340
+	return __pa(vaddr);
341
+}
342
+
343
+static void __init create_pmd_mapping(pmd_t *pmdp,
344
+				      uintptr_t va, phys_addr_t pa,
345
+				      phys_addr_t sz, pgprot_t prot)
346
+{
347
+	pte_t *ptep;
348
+	phys_addr_t pte_phys;
349
+	uintptr_t pmd_idx = pmd_index(va);
350
+
351
+	if (sz == PMD_SIZE) {
352
+		if (pmd_none(pmdp[pmd_idx]))
353
+			pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot);
354
+		return;
355
+	}
356
+
357
+	if (pmd_none(pmdp[pmd_idx])) {
358
+		pte_phys = pt_ops.alloc_pte(va);
359
+		pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE);
360
+		ptep = pt_ops.get_pte_virt(pte_phys);
361
+		memset(ptep, 0, PAGE_SIZE);
362
+	} else {
363
+		pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx]));
364
+		ptep = pt_ops.get_pte_virt(pte_phys);
365
+	}
366
+
367
+	create_pte_mapping(ptep, va, pa, sz, prot);
368
+}
369
+
370
+#define pgd_next_t		pmd_t
371
+#define alloc_pgd_next(__va)	pt_ops.alloc_pmd(__va)
372
+#define get_pgd_next_virt(__pa)	pt_ops.get_pmd_virt(__pa)
373
+#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot)	\
374
+	create_pmd_mapping(__nextp, __va, __pa, __sz, __prot)
375
+#define fixmap_pgd_next		fixmap_pmd
376
+#else
377
+#define pgd_next_t		pte_t
378
+#define alloc_pgd_next(__va)	pt_ops.alloc_pte(__va)
379
+#define get_pgd_next_virt(__pa)	pt_ops.get_pte_virt(__pa)
380
+#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot)	\
381
+	create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
382
+#define fixmap_pgd_next		fixmap_pte
383
+#endif
384
+
385
+void __init create_pgd_mapping(pgd_t *pgdp,
386
+				      uintptr_t va, phys_addr_t pa,
387
+				      phys_addr_t sz, pgprot_t prot)
388
+{
389
+	pgd_next_t *nextp;
390
+	phys_addr_t next_phys;
391
+	uintptr_t pgd_idx = pgd_index(va);
392
+
393
+	if (sz == PGDIR_SIZE) {
394
+		if (pgd_val(pgdp[pgd_idx]) == 0)
395
+			pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot);
396
+		return;
397
+	}
398
+
399
+	if (pgd_val(pgdp[pgd_idx]) == 0) {
400
+		next_phys = alloc_pgd_next(va);
401
+		pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE);
402
+		nextp = get_pgd_next_virt(next_phys);
403
+		memset(nextp, 0, PAGE_SIZE);
404
+	} else {
405
+		next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx]));
406
+		nextp = get_pgd_next_virt(next_phys);
407
+	}
408
+
409
+	create_pgd_next_mapping(nextp, va, pa, sz, prot);
410
+}
411
+
412
+static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)
413
+{
414
+	/* Upgrade to PMD_SIZE mappings whenever possible */
415
+	if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1)))
416
+		return PAGE_SIZE;
417
+
418
+	return PMD_SIZE;
419
+}
420
+
421
+/*
422
+ * setup_vm() is called from head.S with MMU-off.
423
+ *
424
+ * Following requirements should be honoured for setup_vm() to work
425
+ * correctly:
426
+ * 1) It should use PC-relative addressing for accessing kernel symbols.
427
+ *    To achieve this we always use GCC cmodel=medany.
428
+ * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
429
+ *    so disable compiler instrumentation when FTRACE is enabled.
430
+ *
431
+ * Currently, the above requirements are honoured by using custom CFLAGS
432
+ * for init.o in mm/Makefile.
433
+ */
434
+
435
+#ifndef __riscv_cmodel_medany
436
+#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."
437
+#endif
438
+
439
+asmlinkage void __init setup_vm(uintptr_t dtb_pa)
440
+{
441
+	uintptr_t va, pa, end_va;
442
+	uintptr_t load_pa = (uintptr_t)(&_start);
443
+	uintptr_t load_sz = (uintptr_t)(&_end) - load_pa;
444
+	uintptr_t map_size;
445
+#ifndef __PAGETABLE_PMD_FOLDED
446
+	pmd_t fix_bmap_spmd, fix_bmap_epmd;
447
+#endif
448
+
449
+	va_pa_offset = PAGE_OFFSET - load_pa;
450
+	pfn_base = PFN_DOWN(load_pa);
451
+
452
+	/*
453
+	 * Enforce boot alignment requirements of RV32 and
454
+	 * RV64 by only allowing PMD or PGD mappings.
455
+	 */
456
+	map_size = PMD_SIZE;
457
+
458
+	/* Sanity check alignment and size */
459
+	BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0);
460
+	BUG_ON((load_pa % map_size) != 0);
461
+
462
+	pt_ops.alloc_pte = alloc_pte_early;
463
+	pt_ops.get_pte_virt = get_pte_virt_early;
464
+#ifndef __PAGETABLE_PMD_FOLDED
465
+	pt_ops.alloc_pmd = alloc_pmd_early;
466
+	pt_ops.get_pmd_virt = get_pmd_virt_early;
467
+#endif
468
+	/* Setup early PGD for fixmap */
469
+	create_pgd_mapping(early_pg_dir, FIXADDR_START,
470
+			   (uintptr_t)fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);
471
+
472
+#ifndef __PAGETABLE_PMD_FOLDED
473
+	/* Setup fixmap PMD */
474
+	create_pmd_mapping(fixmap_pmd, FIXADDR_START,
475
+			   (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE);
476
+	/* Setup trampoline PGD and PMD */
477
+	create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET,
478
+			   (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE);
479
+	create_pmd_mapping(trampoline_pmd, PAGE_OFFSET,
480
+			   load_pa, PMD_SIZE, PAGE_KERNEL_EXEC);
481
+#else
482
+	/* Setup trampoline PGD */
483
+	create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET,
484
+			   load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC);
485
+#endif
486
+
487
+	/*
488
+	 * Setup early PGD covering entire kernel which will allows
489
+	 * us to reach paging_init(). We map all memory banks later
490
+	 * in setup_vm_final() below.
491
+	 */
492
+	end_va = PAGE_OFFSET + load_sz;
493
+	for (va = PAGE_OFFSET; va < end_va; va += map_size)
494
+		create_pgd_mapping(early_pg_dir, va,
495
+				   load_pa + (va - PAGE_OFFSET),
496
+				   map_size, PAGE_KERNEL_EXEC);
497
+
498
+#ifndef __PAGETABLE_PMD_FOLDED
499
+	/* Setup early PMD for DTB */
500
+	create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA,
501
+			   (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE);
502
+	/* Create two consecutive PMD mappings for FDT early scan */
503
+	pa = dtb_pa & ~(PMD_SIZE - 1);
504
+	create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA,
505
+			   pa, PMD_SIZE, PAGE_KERNEL);
506
+	create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE,
507
+			   pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL);
508
+	dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1));
509
+#else
510
+	/* Create two consecutive PGD mappings for FDT early scan */
511
+	pa = dtb_pa & ~(PGDIR_SIZE - 1);
512
+	create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA,
513
+			   pa, PGDIR_SIZE, PAGE_KERNEL);
514
+	create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE,
515
+			   pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL);
516
+	dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1));
517
+#endif
518
+	dtb_early_pa = dtb_pa;
519
+
520
+	/*
521
+	 * Bootime fixmap only can handle PMD_SIZE mapping. Thus, boot-ioremap
522
+	 * range can not span multiple pmds.
523
+	 */
524
+	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
525
+		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
526
+
527
+#ifndef __PAGETABLE_PMD_FOLDED
528
+	/*
529
+	 * Early ioremap fixmap is already created as it lies within first 2MB
530
+	 * of fixmap region. We always map PMD_SIZE. Thus, both FIX_BTMAP_END
531
+	 * FIX_BTMAP_BEGIN should lie in the same pmd. Verify that and warn
532
+	 * the user if not.
533
+	 */
534
+	fix_bmap_spmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_BEGIN))];
535
+	fix_bmap_epmd = fixmap_pmd[pmd_index(__fix_to_virt(FIX_BTMAP_END))];
536
+	if (pmd_val(fix_bmap_spmd) != pmd_val(fix_bmap_epmd)) {
537
+		WARN_ON(1);
538
+		pr_warn("fixmap btmap start [%08lx] != end [%08lx]\n",
539
+			pmd_val(fix_bmap_spmd), pmd_val(fix_bmap_epmd));
540
+		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
541
+			fix_to_virt(FIX_BTMAP_BEGIN));
542
+		pr_warn("fix_to_virt(FIX_BTMAP_END):   %08lx\n",
543
+			fix_to_virt(FIX_BTMAP_END));
544
+
545
+		pr_warn("FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
546
+		pr_warn("FIX_BTMAP_BEGIN:     %d\n", FIX_BTMAP_BEGIN);
547
+	}
548
+#endif
549
+}
550
+
551
+static void __init setup_vm_final(void)
552
+{
553
+	uintptr_t va, map_size;
554
+	phys_addr_t pa, start, end;
555
+	u64 i;
556
+
557
+	/**
558
+	 * MMU is enabled at this point. But page table setup is not complete yet.
559
+	 * fixmap page table alloc functions should be used at this point
560
+	 */
561
+	pt_ops.alloc_pte = alloc_pte_fixmap;
562
+	pt_ops.get_pte_virt = get_pte_virt_fixmap;
563
+#ifndef __PAGETABLE_PMD_FOLDED
564
+	pt_ops.alloc_pmd = alloc_pmd_fixmap;
565
+	pt_ops.get_pmd_virt = get_pmd_virt_fixmap;
566
+#endif
567
+	/* Setup swapper PGD for fixmap */
568
+	create_pgd_mapping(swapper_pg_dir, FIXADDR_START,
569
+			   __pa_symbol(fixmap_pgd_next),
570
+			   PGDIR_SIZE, PAGE_TABLE);
571
+
572
+	/* Map all memory banks */
573
+	for_each_mem_range(i, &start, &end) {
574
+		if (start >= end)
575
+			break;
576
+		if (start <= __pa(PAGE_OFFSET) &&
577
+		    __pa(PAGE_OFFSET) < end)
578
+			start = __pa(PAGE_OFFSET);
579
+
580
+		map_size = best_map_size(start, end - start);
581
+		for (pa = start; pa < end; pa += map_size) {
582
+			va = (uintptr_t)__va(pa);
583
+			create_pgd_mapping(swapper_pg_dir, va, pa,
584
+					   map_size, PAGE_KERNEL_EXEC);
585
+		}
586
+	}
587
+
588
+	/* Clear fixmap PTE and PMD mappings */
589
+	clear_fixmap(FIX_PTE);
590
+	clear_fixmap(FIX_PMD);
591
+
592
+	/* Move to swapper page table */
593
+	csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE);
594
+	local_flush_tlb_all();
595
+
596
+	/* generic page allocation functions must be used to setup page table */
597
+	pt_ops.alloc_pte = alloc_pte_late;
598
+	pt_ops.get_pte_virt = get_pte_virt_late;
599
+#ifndef __PAGETABLE_PMD_FOLDED
600
+	pt_ops.alloc_pmd = alloc_pmd_late;
601
+	pt_ops.get_pmd_virt = get_pmd_virt_late;
602
+#endif
603
+}
604
+#else
605
+asmlinkage void __init setup_vm(uintptr_t dtb_pa)
606
+{
607
+#ifdef CONFIG_BUILTIN_DTB
608
+	dtb_early_va = soc_lookup_builtin_dtb();
609
+	if (!dtb_early_va) {
610
+		/* Fallback to first available DTS */
611
+		dtb_early_va = (void *) __dtb_start;
612
+	}
613
+#else
614
+	dtb_early_va = (void *)dtb_pa;
615
+#endif
616
+	dtb_early_pa = dtb_pa;
617
+}
618
+
619
+static inline void setup_vm_final(void)
620
+{
621
+}
622
+#endif /* CONFIG_MMU */
623
+
624
+#ifdef CONFIG_STRICT_KERNEL_RWX
625
+void mark_rodata_ro(void)
626
+{
627
+	unsigned long text_start = (unsigned long)_text;
628
+	unsigned long text_end = (unsigned long)_etext;
629
+	unsigned long rodata_start = (unsigned long)__start_rodata;
630
+	unsigned long data_start = (unsigned long)_data;
631
+	unsigned long max_low = (unsigned long)(__va(PFN_PHYS(max_low_pfn)));
632
+
633
+	set_memory_ro(text_start, (text_end - text_start) >> PAGE_SHIFT);
634
+	set_memory_ro(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT);
635
+	set_memory_nx(rodata_start, (data_start - rodata_start) >> PAGE_SHIFT);
636
+	set_memory_nx(data_start, (max_low - data_start) >> PAGE_SHIFT);
637
+
638
+	debug_checkwx();
639
+}
640
+#endif
641
+
642
+static void __init resource_init(void)
643
+{
644
+	struct memblock_region *region;
645
+
646
+	for_each_mem_region(region) {
647
+		struct resource *res;
648
+
649
+		res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES);
650
+		if (!res)
651
+			panic("%s: Failed to allocate %zu bytes\n", __func__,
652
+			      sizeof(struct resource));
653
+
654
+		if (memblock_is_nomap(region)) {
655
+			res->name = "reserved";
656
+			res->flags = IORESOURCE_MEM;
657
+		} else {
658
+			res->name = "System RAM";
659
+			res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
660
+		}
661
+		res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region));
662
+		res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1;
663
+
664
+		request_resource(&iomem_resource, res);
665
+	}
666
+}
667
+
668
+void __init paging_init(void)
669
+{
670
+	setup_vm_final();
671
+	setup_zero_page();
672
+}
673
+
674
+void __init misc_mem_init(void)
675
+{
676
+	sparse_init();
677
+	zone_sizes_init();
678
+	resource_init();
679
+}
680
+
681
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
682
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
683
+			       struct vmem_altmap *altmap)
684
+{
685
+	return vmemmap_populate_basepages(start, end, node, NULL);
686
+}
687
+#endif