.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * arch/arm64/mm/hugetlbpage.c |
---|
3 | 4 | * |
---|
4 | 5 | * Copyright (C) 2013 Linaro Ltd. |
---|
5 | 6 | * |
---|
6 | 7 | * Based on arch/x86/mm/hugetlbpage.c. |
---|
7 | | - * |
---|
8 | | - * This program is free software; you can redistribute it and/or modify |
---|
9 | | - * it under the terms of the GNU General Public License version 2 as |
---|
10 | | - * published by the Free Software Foundation. |
---|
11 | | - * |
---|
12 | | - * This program is distributed in the hope that it will be useful, |
---|
13 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
14 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
15 | | - * GNU General Public License for more details. |
---|
16 | 8 | */ |
---|
17 | 9 | |
---|
18 | 10 | #include <linux/init.h> |
---|
.. | .. |
---|
25 | 17 | #include <asm/mman.h> |
---|
26 | 18 | #include <asm/tlb.h> |
---|
27 | 19 | #include <asm/tlbflush.h> |
---|
28 | | -#include <asm/pgalloc.h> |
---|
| 20 | + |
---|
| 21 | +/* |
---|
| 22 | + * HugeTLB Support Matrix |
---|
| 23 | + * |
---|
| 24 | + * --------------------------------------------------- |
---|
| 25 | + * | Page Size | CONT PTE | PMD | CONT PMD | PUD | |
---|
| 26 | + * --------------------------------------------------- |
---|
| 27 | + * | 4K | 64K | 2M | 32M | 1G | |
---|
| 28 | + * | 16K | 2M | 32M | 1G | | |
---|
| 29 | + * | 64K | 2M | 512M | 16G | | |
---|
| 30 | + * --------------------------------------------------- |
---|
| 31 | + */ |
---|
| 32 | + |
---|
| 33 | +/* |
---|
| 34 | + * Reserve CMA areas for the largest supported gigantic |
---|
| 35 | + * huge page when requested. Any other smaller gigantic |
---|
| 36 | + * huge pages could still be served from those areas. |
---|
| 37 | + */ |
---|
| 38 | +#ifdef CONFIG_CMA |
---|
| 39 | +void __init arm64_hugetlb_cma_reserve(void) |
---|
| 40 | +{ |
---|
| 41 | + int order; |
---|
| 42 | + |
---|
| 43 | +#ifdef CONFIG_ARM64_4K_PAGES |
---|
| 44 | + order = PUD_SHIFT - PAGE_SHIFT; |
---|
| 45 | +#else |
---|
| 46 | + order = CONT_PMD_SHIFT - PAGE_SHIFT; |
---|
| 47 | +#endif |
---|
| 48 | + /* |
---|
| 49 | + * HugeTLB CMA reservation is required for gigantic |
---|
| 50 | + * huge pages which could not be allocated via the |
---|
| 51 | + * page allocator. Just warn if there is any change |
---|
| 52 | + * breaking this assumption. |
---|
| 53 | + */ |
---|
| 54 | + WARN_ON(order <= MAX_ORDER); |
---|
| 55 | + hugetlb_cma_reserve(order); |
---|
| 56 | +} |
---|
| 57 | +#endif /* CONFIG_CMA */ |
---|
| 58 | + |
---|
| 59 | +#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION |
---|
| 60 | +bool arch_hugetlb_migration_supported(struct hstate *h) |
---|
| 61 | +{ |
---|
| 62 | + size_t pagesize = huge_page_size(h); |
---|
| 63 | + |
---|
| 64 | + switch (pagesize) { |
---|
| 65 | +#ifdef CONFIG_ARM64_4K_PAGES |
---|
| 66 | + case PUD_SIZE: |
---|
| 67 | +#endif |
---|
| 68 | + case PMD_SIZE: |
---|
| 69 | + case CONT_PMD_SIZE: |
---|
| 70 | + case CONT_PTE_SIZE: |
---|
| 71 | + return true; |
---|
| 72 | + } |
---|
| 73 | + pr_warn("%s: unrecognized huge page size 0x%lx\n", |
---|
| 74 | + __func__, pagesize); |
---|
| 75 | + return false; |
---|
| 76 | +} |
---|
| 77 | +#endif |
---|
29 | 78 | |
---|
30 | 79 | int pmd_huge(pmd_t pmd) |
---|
31 | 80 | { |
---|
.. | .. |
---|
55 | 104 | pte_t *ptep, size_t *pgsize) |
---|
56 | 105 | { |
---|
57 | 106 | pgd_t *pgdp = pgd_offset(mm, addr); |
---|
| 107 | + p4d_t *p4dp; |
---|
58 | 108 | pud_t *pudp; |
---|
59 | 109 | pmd_t *pmdp; |
---|
60 | 110 | |
---|
61 | 111 | *pgsize = PAGE_SIZE; |
---|
62 | | - pudp = pud_offset(pgdp, addr); |
---|
| 112 | + p4dp = p4d_offset(pgdp, addr); |
---|
| 113 | + pudp = pud_offset(p4dp, addr); |
---|
63 | 114 | pmdp = pmd_offset(pudp, addr); |
---|
64 | 115 | if ((pte_t *)pmdp == ptep) { |
---|
65 | 116 | *pgsize = PMD_SIZE; |
---|
.. | .. |
---|
201 | 252 | set_pte(ptep, pte); |
---|
202 | 253 | } |
---|
203 | 254 | |
---|
204 | | -pte_t *huge_pte_alloc(struct mm_struct *mm, |
---|
| 255 | +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, |
---|
205 | 256 | unsigned long addr, unsigned long sz) |
---|
206 | 257 | { |
---|
207 | 258 | pgd_t *pgdp; |
---|
| 259 | + p4d_t *p4dp; |
---|
208 | 260 | pud_t *pudp; |
---|
209 | 261 | pmd_t *pmdp; |
---|
210 | 262 | pte_t *ptep = NULL; |
---|
211 | 263 | |
---|
212 | 264 | pgdp = pgd_offset(mm, addr); |
---|
213 | | - pudp = pud_alloc(mm, pgdp, addr); |
---|
| 265 | + p4dp = p4d_offset(pgdp, addr); |
---|
| 266 | + pudp = pud_alloc(mm, p4dp, addr); |
---|
214 | 267 | if (!pudp) |
---|
215 | 268 | return NULL; |
---|
216 | 269 | |
---|
217 | 270 | if (sz == PUD_SIZE) { |
---|
218 | 271 | ptep = (pte_t *)pudp; |
---|
219 | | - } else if (sz == (PAGE_SIZE * CONT_PTES)) { |
---|
| 272 | + } else if (sz == (CONT_PTE_SIZE)) { |
---|
220 | 273 | pmdp = pmd_alloc(mm, pudp, addr); |
---|
221 | 274 | if (!pmdp) |
---|
222 | 275 | return NULL; |
---|
.. | .. |
---|
231 | 284 | */ |
---|
232 | 285 | ptep = pte_alloc_map(mm, pmdp, addr); |
---|
233 | 286 | } else if (sz == PMD_SIZE) { |
---|
234 | | - if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && |
---|
235 | | - pud_none(READ_ONCE(*pudp))) |
---|
236 | | - ptep = huge_pmd_share(mm, addr, pudp); |
---|
| 287 | + if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp))) |
---|
| 288 | + ptep = huge_pmd_share(mm, vma, addr, pudp); |
---|
237 | 289 | else |
---|
238 | 290 | ptep = (pte_t *)pmd_alloc(mm, pudp, addr); |
---|
239 | | - } else if (sz == (PMD_SIZE * CONT_PMDS)) { |
---|
| 291 | + } else if (sz == (CONT_PMD_SIZE)) { |
---|
240 | 292 | pmdp = pmd_alloc(mm, pudp, addr); |
---|
241 | 293 | WARN_ON(addr & (sz - 1)); |
---|
242 | 294 | return (pte_t *)pmdp; |
---|
.. | .. |
---|
249 | 301 | unsigned long addr, unsigned long sz) |
---|
250 | 302 | { |
---|
251 | 303 | pgd_t *pgdp; |
---|
| 304 | + p4d_t *p4dp; |
---|
252 | 305 | pud_t *pudp, pud; |
---|
253 | 306 | pmd_t *pmdp, pmd; |
---|
254 | 307 | |
---|
.. | .. |
---|
256 | 309 | if (!pgd_present(READ_ONCE(*pgdp))) |
---|
257 | 310 | return NULL; |
---|
258 | 311 | |
---|
259 | | - pudp = pud_offset(pgdp, addr); |
---|
| 312 | + p4dp = p4d_offset(pgdp, addr); |
---|
| 313 | + if (!p4d_present(READ_ONCE(*p4dp))) |
---|
| 314 | + return NULL; |
---|
| 315 | + |
---|
| 316 | + pudp = pud_offset(p4dp, addr); |
---|
260 | 317 | pud = READ_ONCE(*pudp); |
---|
261 | 318 | if (sz != PUD_SIZE && pud_none(pud)) |
---|
262 | 319 | return NULL; |
---|
.. | .. |
---|
431 | 488 | clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); |
---|
432 | 489 | } |
---|
433 | 490 | |
---|
434 | | -static __init int setup_hugepagesz(char *opt) |
---|
| 491 | +static int __init hugetlbpage_init(void) |
---|
435 | 492 | { |
---|
436 | | - unsigned long ps = memparse(opt, &opt); |
---|
| 493 | +#ifdef CONFIG_ARM64_4K_PAGES |
---|
| 494 | + hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); |
---|
| 495 | +#endif |
---|
| 496 | + hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT); |
---|
| 497 | + hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); |
---|
| 498 | + hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT); |
---|
437 | 499 | |
---|
438 | | - switch (ps) { |
---|
| 500 | + return 0; |
---|
| 501 | +} |
---|
| 502 | +arch_initcall(hugetlbpage_init); |
---|
| 503 | + |
---|
| 504 | +bool __init arch_hugetlb_valid_size(unsigned long size) |
---|
| 505 | +{ |
---|
| 506 | + switch (size) { |
---|
439 | 507 | #ifdef CONFIG_ARM64_4K_PAGES |
---|
440 | 508 | case PUD_SIZE: |
---|
441 | 509 | #endif |
---|
442 | | - case PMD_SIZE * CONT_PMDS: |
---|
| 510 | + case CONT_PMD_SIZE: |
---|
443 | 511 | case PMD_SIZE: |
---|
444 | | - case PAGE_SIZE * CONT_PTES: |
---|
445 | | - hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); |
---|
446 | | - return 1; |
---|
| 512 | + case CONT_PTE_SIZE: |
---|
| 513 | + return true; |
---|
447 | 514 | } |
---|
448 | 515 | |
---|
449 | | - hugetlb_bad_size(); |
---|
450 | | - pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); |
---|
451 | | - return 0; |
---|
| 516 | + return false; |
---|
452 | 517 | } |
---|
453 | | -__setup("hugepagesz=", setup_hugepagesz); |
---|
454 | | - |
---|
455 | | -#ifdef CONFIG_ARM64_64K_PAGES |
---|
456 | | -static __init int add_default_hugepagesz(void) |
---|
457 | | -{ |
---|
458 | | - if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) |
---|
459 | | - hugetlb_add_hstate(CONT_PTE_SHIFT); |
---|
460 | | - return 0; |
---|
461 | | -} |
---|
462 | | -arch_initcall(add_default_hugepagesz); |
---|
463 | | -#endif |
---|