update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/arch/arm64/mm/hugetlbpage.c
..
..
@@ -1,18 +1,10 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /*
2
3
  * arch/arm64/mm/hugetlbpage.c
3
4
  *
4
5
  * Copyright (C) 2013 Linaro Ltd.
5
6
  *
6
7
  * Based on arch/x86/mm/hugetlbpage.c.
7
- *
8
- * This program is free software; you can redistribute it and/or modify
9
- * it under the terms of the GNU General Public License version 2 as
10
- * published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope that it will be useful,
13
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
- * GNU General Public License for more details.
16
8
  */
17
9
 
18
10
 #include <linux/init.h>
..
..
@@ -25,7 +17,64 @@
25
17
 #include <asm/mman.h>
26
18
 #include <asm/tlb.h>
27
19
 #include <asm/tlbflush.h>
28
-#include <asm/pgalloc.h>
20
+
21
+/*
22
+ * HugeTLB Support Matrix
23
+ *
24
+ * ---------------------------------------------------
25
+ * | Page Size | CONT PTE |  PMD  | CONT PMD |  PUD  |
26
+ * ---------------------------------------------------
27
+ * |     4K    |   64K    |   2M  |    32M   |   1G  |
28
+ * |    16K    |    2M    |  32M  |     1G   |       |
29
+ * |    64K    |    2M    | 512M  |    16G   |       |
30
+ * ---------------------------------------------------
31
+ */
32
+
33
+/*
34
+ * Reserve CMA areas for the largest supported gigantic
35
+ * huge page when requested. Any other smaller gigantic
36
+ * huge pages could still be served from those areas.
37
+ */
38
+#ifdef CONFIG_CMA
39
+void __init arm64_hugetlb_cma_reserve(void)
40
+{
41
+	int order;
42
+
43
+#ifdef CONFIG_ARM64_4K_PAGES
44
+	order = PUD_SHIFT - PAGE_SHIFT;
45
+#else
46
+	order = CONT_PMD_SHIFT - PAGE_SHIFT;
47
+#endif
48
+	/*
49
+	 * HugeTLB CMA reservation is required for gigantic
50
+	 * huge pages which could not be allocated via the
51
+	 * page allocator. Just warn if there is any change
52
+	 * breaking this assumption.
53
+	 */
54
+	WARN_ON(order <= MAX_ORDER);
55
+	hugetlb_cma_reserve(order);
56
+}
57
+#endif /* CONFIG_CMA */
58
+
59
+#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
60
+bool arch_hugetlb_migration_supported(struct hstate *h)
61
+{
62
+	size_t pagesize = huge_page_size(h);
63
+
64
+	switch (pagesize) {
65
+#ifdef CONFIG_ARM64_4K_PAGES
66
+	case PUD_SIZE:
67
+#endif
68
+	case PMD_SIZE:
69
+	case CONT_PMD_SIZE:
70
+	case CONT_PTE_SIZE:
71
+		return true;
72
+	}
73
+	pr_warn("%s: unrecognized huge page size 0x%lx\n",
74
+			__func__, pagesize);
75
+	return false;
76
+}
77
+#endif
29
78
 
30
79
 int pmd_huge(pmd_t pmd)
31
80
 {
..
..
@@ -55,11 +104,13 @@
55
104
 			   pte_t *ptep, size_t *pgsize)
56
105
 {
57
106
 	pgd_t *pgdp = pgd_offset(mm, addr);
107
+	p4d_t *p4dp;
58
108
 	pud_t *pudp;
59
109
 	pmd_t *pmdp;
60
110
 
61
111
 	*pgsize = PAGE_SIZE;
62
-	pudp = pud_offset(pgdp, addr);
112
+	p4dp = p4d_offset(pgdp, addr);
113
+	pudp = pud_offset(p4dp, addr);
63
114
 	pmdp = pmd_offset(pudp, addr);
64
115
 	if ((pte_t *)pmdp == ptep) {
65
116
 		*pgsize = PMD_SIZE;
..
..
@@ -201,22 +252,24 @@
201
252
 		set_pte(ptep, pte);
202
253
 }
203
254
 
204
-pte_t *huge_pte_alloc(struct mm_struct *mm,
255
+pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
205
256
 		      unsigned long addr, unsigned long sz)
206
257
 {
207
258
 	pgd_t *pgdp;
259
+	p4d_t *p4dp;
208
260
 	pud_t *pudp;
209
261
 	pmd_t *pmdp;
210
262
 	pte_t *ptep = NULL;
211
263
 
212
264
 	pgdp = pgd_offset(mm, addr);
213
-	pudp = pud_alloc(mm, pgdp, addr);
265
+	p4dp = p4d_offset(pgdp, addr);
266
+	pudp = pud_alloc(mm, p4dp, addr);
214
267
 	if (!pudp)
215
268
 		return NULL;
216
269
 
217
270
 	if (sz == PUD_SIZE) {
218
271
 		ptep = (pte_t *)pudp;
219
-	} else if (sz == (PAGE_SIZE * CONT_PTES)) {
272
+	} else if (sz == (CONT_PTE_SIZE)) {
220
273
 		pmdp = pmd_alloc(mm, pudp, addr);
221
274
 		if (!pmdp)
222
275
 			return NULL;
..
..
@@ -231,12 +284,11 @@
231
284
 		 */
232
285
 		ptep = pte_alloc_map(mm, pmdp, addr);
233
286
 	} else if (sz == PMD_SIZE) {
234
-		if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
235
-		    pud_none(READ_ONCE(*pudp)))
236
-			ptep = huge_pmd_share(mm, addr, pudp);
287
+		if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
288
+			ptep = huge_pmd_share(mm, vma, addr, pudp);
237
289
 		else
238
290
 			ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
239
-	} else if (sz == (PMD_SIZE * CONT_PMDS)) {
291
+	} else if (sz == (CONT_PMD_SIZE)) {
240
292
 		pmdp = pmd_alloc(mm, pudp, addr);
241
293
 		WARN_ON(addr & (sz - 1));
242
294
 		return (pte_t *)pmdp;
..
..
@@ -249,6 +301,7 @@
249
301
 		       unsigned long addr, unsigned long sz)
250
302
 {
251
303
 	pgd_t *pgdp;
304
+	p4d_t *p4dp;
252
305
 	pud_t *pudp, pud;
253
306
 	pmd_t *pmdp, pmd;
254
307
 
..
..
@@ -256,7 +309,11 @@
256
309
 	if (!pgd_present(READ_ONCE(*pgdp)))
257
310
 		return NULL;
258
311
 
259
-	pudp = pud_offset(pgdp, addr);
312
+	p4dp = p4d_offset(pgdp, addr);
313
+	if (!p4d_present(READ_ONCE(*p4dp)))
314
+		return NULL;
315
+
316
+	pudp = pud_offset(p4dp, addr);
260
317
 	pud = READ_ONCE(*pudp);
261
318
 	if (sz != PUD_SIZE && pud_none(pud))
262
319
 		return NULL;
..
..
@@ -431,33 +488,30 @@
431
488
 	clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
432
489
 }
433
490
 
434
-static __init int setup_hugepagesz(char *opt)
491
+static int __init hugetlbpage_init(void)
435
492
 {
436
-	unsigned long ps = memparse(opt, &opt);
493
+#ifdef CONFIG_ARM64_4K_PAGES
494
+	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
495
+#endif
496
+	hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
497
+	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
498
+	hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
437
499
 
438
-	switch (ps) {
500
+	return 0;
501
+}
502
+arch_initcall(hugetlbpage_init);
503
+
504
+bool __init arch_hugetlb_valid_size(unsigned long size)
505
+{
506
+	switch (size) {
439
507
 #ifdef CONFIG_ARM64_4K_PAGES
440
508
 	case PUD_SIZE:
441
509
 #endif
442
-	case PMD_SIZE * CONT_PMDS:
510
+	case CONT_PMD_SIZE:
443
511
 	case PMD_SIZE:
444
-	case PAGE_SIZE * CONT_PTES:
445
-		hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT);
446
-		return 1;
512
+	case CONT_PTE_SIZE:
513
+		return true;
447
514
 	}
448
515
 
449
-	hugetlb_bad_size();
450
-	pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
451
-	return 0;
516
+	return false;
452
517
 }
453
-__setup("hugepagesz=", setup_hugepagesz);
454
-
455
-#ifdef CONFIG_ARM64_64K_PAGES
456
-static __init int add_default_hugepagesz(void)
457
-{
458
-	if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
459
-		hugetlb_add_hstate(CONT_PTE_SHIFT);
460
-	return 0;
461
-}
462
-arch_initcall(add_default_hugepagesz);
463
-#endif