hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da
kernel/arch/arm64/mm/hugetlbpage.c
....@@ -1,18 +1,10 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23 * arch/arm64/mm/hugetlbpage.c
34 *
45 * Copyright (C) 2013 Linaro Ltd.
56 *
67 * Based on arch/x86/mm/hugetlbpage.c.
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
- * GNU General Public License for more details.
168 */
179
1810 #include <linux/init.h>
....@@ -25,7 +17,64 @@
2517 #include <asm/mman.h>
2618 #include <asm/tlb.h>
2719 #include <asm/tlbflush.h>
28
-#include <asm/pgalloc.h>
20
+
21
+/*
22
+ * HugeTLB Support Matrix
23
+ *
24
+ * ---------------------------------------------------
25
+ * | Page Size | CONT PTE | PMD | CONT PMD | PUD |
26
+ * ---------------------------------------------------
27
+ * | 4K | 64K | 2M | 32M | 1G |
28
+ * | 16K | 2M | 32M | 1G | |
29
+ * | 64K | 2M | 512M | 16G | |
30
+ * ---------------------------------------------------
31
+ */
32
+
33
+/*
34
+ * Reserve CMA areas for the largest supported gigantic
35
+ * huge page when requested. Any other smaller gigantic
36
+ * huge pages could still be served from those areas.
37
+ */
38
+#ifdef CONFIG_CMA
39
+void __init arm64_hugetlb_cma_reserve(void)
40
+{
41
+ int order;
42
+
43
+#ifdef CONFIG_ARM64_4K_PAGES
44
+ order = PUD_SHIFT - PAGE_SHIFT;
45
+#else
46
+ order = CONT_PMD_SHIFT - PAGE_SHIFT;
47
+#endif
48
+ /*
49
+ * HugeTLB CMA reservation is required for gigantic
50
+ * huge pages which could not be allocated via the
51
+ * page allocator. Just warn if there is any change
52
+ * breaking this assumption.
53
+ */
54
+ WARN_ON(order <= MAX_ORDER);
55
+ hugetlb_cma_reserve(order);
56
+}
57
+#endif /* CONFIG_CMA */
58
+
59
+#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
60
+bool arch_hugetlb_migration_supported(struct hstate *h)
61
+{
62
+ size_t pagesize = huge_page_size(h);
63
+
64
+ switch (pagesize) {
65
+#ifdef CONFIG_ARM64_4K_PAGES
66
+ case PUD_SIZE:
67
+#endif
68
+ case PMD_SIZE:
69
+ case CONT_PMD_SIZE:
70
+ case CONT_PTE_SIZE:
71
+ return true;
72
+ }
73
+ pr_warn("%s: unrecognized huge page size 0x%lx\n",
74
+ __func__, pagesize);
75
+ return false;
76
+}
77
+#endif
2978
3079 int pmd_huge(pmd_t pmd)
3180 {
....@@ -55,11 +104,13 @@
55104 pte_t *ptep, size_t *pgsize)
56105 {
57106 pgd_t *pgdp = pgd_offset(mm, addr);
107
+ p4d_t *p4dp;
58108 pud_t *pudp;
59109 pmd_t *pmdp;
60110
61111 *pgsize = PAGE_SIZE;
62
- pudp = pud_offset(pgdp, addr);
112
+ p4dp = p4d_offset(pgdp, addr);
113
+ pudp = pud_offset(p4dp, addr);
63114 pmdp = pmd_offset(pudp, addr);
64115 if ((pte_t *)pmdp == ptep) {
65116 *pgsize = PMD_SIZE;
....@@ -201,22 +252,24 @@
201252 set_pte(ptep, pte);
202253 }
203254
204
-pte_t *huge_pte_alloc(struct mm_struct *mm,
255
+pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
205256 unsigned long addr, unsigned long sz)
206257 {
207258 pgd_t *pgdp;
259
+ p4d_t *p4dp;
208260 pud_t *pudp;
209261 pmd_t *pmdp;
210262 pte_t *ptep = NULL;
211263
212264 pgdp = pgd_offset(mm, addr);
213
- pudp = pud_alloc(mm, pgdp, addr);
265
+ p4dp = p4d_offset(pgdp, addr);
266
+ pudp = pud_alloc(mm, p4dp, addr);
214267 if (!pudp)
215268 return NULL;
216269
217270 if (sz == PUD_SIZE) {
218271 ptep = (pte_t *)pudp;
219
- } else if (sz == (PAGE_SIZE * CONT_PTES)) {
272
+ } else if (sz == (CONT_PTE_SIZE)) {
220273 pmdp = pmd_alloc(mm, pudp, addr);
221274 if (!pmdp)
222275 return NULL;
....@@ -231,12 +284,11 @@
231284 */
232285 ptep = pte_alloc_map(mm, pmdp, addr);
233286 } else if (sz == PMD_SIZE) {
234
- if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
235
- pud_none(READ_ONCE(*pudp)))
236
- ptep = huge_pmd_share(mm, addr, pudp);
287
+ if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
288
+ ptep = huge_pmd_share(mm, vma, addr, pudp);
237289 else
238290 ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
239
- } else if (sz == (PMD_SIZE * CONT_PMDS)) {
291
+ } else if (sz == (CONT_PMD_SIZE)) {
240292 pmdp = pmd_alloc(mm, pudp, addr);
241293 WARN_ON(addr & (sz - 1));
242294 return (pte_t *)pmdp;
....@@ -249,6 +301,7 @@
249301 unsigned long addr, unsigned long sz)
250302 {
251303 pgd_t *pgdp;
304
+ p4d_t *p4dp;
252305 pud_t *pudp, pud;
253306 pmd_t *pmdp, pmd;
254307
....@@ -256,7 +309,11 @@
256309 if (!pgd_present(READ_ONCE(*pgdp)))
257310 return NULL;
258311
259
- pudp = pud_offset(pgdp, addr);
312
+ p4dp = p4d_offset(pgdp, addr);
313
+ if (!p4d_present(READ_ONCE(*p4dp)))
314
+ return NULL;
315
+
316
+ pudp = pud_offset(p4dp, addr);
260317 pud = READ_ONCE(*pudp);
261318 if (sz != PUD_SIZE && pud_none(pud))
262319 return NULL;
....@@ -431,33 +488,30 @@
431488 clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
432489 }
433490
434
-static __init int setup_hugepagesz(char *opt)
491
+static int __init hugetlbpage_init(void)
435492 {
436
- unsigned long ps = memparse(opt, &opt);
493
+#ifdef CONFIG_ARM64_4K_PAGES
494
+ hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
495
+#endif
496
+ hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
497
+ hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
498
+ hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
437499
438
- switch (ps) {
500
+ return 0;
501
+}
502
+arch_initcall(hugetlbpage_init);
503
+
504
+bool __init arch_hugetlb_valid_size(unsigned long size)
505
+{
506
+ switch (size) {
439507 #ifdef CONFIG_ARM64_4K_PAGES
440508 case PUD_SIZE:
441509 #endif
442
- case PMD_SIZE * CONT_PMDS:
510
+ case CONT_PMD_SIZE:
443511 case PMD_SIZE:
444
- case PAGE_SIZE * CONT_PTES:
445
- hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT);
446
- return 1;
512
+ case CONT_PTE_SIZE:
513
+ return true;
447514 }
448515
449
- hugetlb_bad_size();
450
- pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
451
- return 0;
516
+ return false;
452517 }
453
-__setup("hugepagesz=", setup_hugepagesz);
454
-
455
-#ifdef CONFIG_ARM64_64K_PAGES
456
-static __init int add_default_hugepagesz(void)
457
-{
458
- if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
459
- hugetlb_add_hstate(CONT_PTE_SHIFT);
460
- return 0;
461
-}
462
-arch_initcall(add_default_hugepagesz);
463
-#endif