forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 958e46acc8e900e8569dd467c1af9b8d2d019394 
 kernel/arch/powerpc/include/asm/book3s/64/mmu-hash.h
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 #ifndef _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
23 #define _ASM_POWERPC_BOOK3S_64_MMU_HASH_H_
34 /*
....@@ -5,11 +6,6 @@
56  *
67  * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com>
78  *   PPC64 rework.
8
- *
9
- * This program is free software; you can redistribute it and/or
10
- * modify it under the terms of the GNU General Public License
11
- * as published by the Free Software Foundation; either version
12
- * 2 of the License, or (at your option) any later version.
139  */
1410 
1511 #include <asm/page.h>
....@@ -23,14 +19,14 @@
2319  */
2420 #include <asm/book3s/64/pgtable.h>
2521 #include <asm/bug.h>
26
-#include <asm/processor.h>
22
+#include <asm/task_size_64.h>
2723 #include <asm/cpu_has_feature.h>
2824 
2925 /*
3026  * SLB
3127  */
3228 
33
-#define SLB_NUM_BOLTED		3
29
+#define SLB_NUM_BOLTED		2
3430 #define SLB_CACHE_ENTRIES	8
3531 #define SLB_MIN_SIZE		32
3632 
....@@ -90,8 +86,8 @@
9086 #define HPTE_R_PP0		ASM_CONST(0x8000000000000000)
9187 #define HPTE_R_TS		ASM_CONST(0x4000000000000000)
9288 #define HPTE_R_KEY_HI		ASM_CONST(0x3000000000000000)
93
-#define HPTE_R_KEY_BIT0		ASM_CONST(0x2000000000000000)
94
-#define HPTE_R_KEY_BIT1		ASM_CONST(0x1000000000000000)
89
+#define HPTE_R_KEY_BIT4		ASM_CONST(0x2000000000000000)
90
+#define HPTE_R_KEY_BIT3		ASM_CONST(0x1000000000000000)
9591 #define HPTE_R_RPN_SHIFT	12
9692 #define HPTE_R_RPN		ASM_CONST(0x0ffffffffffff000)
9793 #define HPTE_R_RPN_3_0		ASM_CONST(0x01fffffffffff000)
....@@ -107,8 +103,8 @@
107103 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
108104 #define HPTE_R_KEY_LO		ASM_CONST(0x0000000000000e00)
109105 #define HPTE_R_KEY_BIT2		ASM_CONST(0x0000000000000800)
110
-#define HPTE_R_KEY_BIT3		ASM_CONST(0x0000000000000400)
111
-#define HPTE_R_KEY_BIT4		ASM_CONST(0x0000000000000200)
106
+#define HPTE_R_KEY_BIT1		ASM_CONST(0x0000000000000400)
107
+#define HPTE_R_KEY_BIT0		ASM_CONST(0x0000000000000200)
112108 #define HPTE_R_KEY		(HPTE_R_KEY_LO | HPTE_R_KEY_HI)
113109 
114110 #define HPTE_V_1TB_SEG		ASM_CONST(0x4000000000000000)
....@@ -201,6 +197,18 @@
201197 	if (mmu_psize_defs[mmu_psize].shift)
202198 		return mmu_psize_defs[mmu_psize].shift;
203199 	BUG();
200
+}
201
+
202
+static inline unsigned int ap_to_shift(unsigned long ap)
203
+{
204
+	int psize;
205
+
206
+	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
207
+		if (mmu_psize_defs[psize].ap == ap)
208
+			return mmu_psize_defs[psize].shift;
209
+	}
210
+
211
+	return -1;
204212 }
205213 
206214 static inline unsigned long get_sllp_encoding(int psize)
....@@ -487,6 +495,8 @@
487495 extern void pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages);
488496 extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr);
489497 
498
+extern void hash__setup_new_exec(void);
499
+
490500 #ifdef CONFIG_PPC_PSERIES
491501 void hpte_init_pseries(void);
492502 #else
....@@ -495,11 +505,18 @@
495505 
496506 extern void hpte_init_native(void);
497507 
508
+struct slb_entry {
509
+	u64	esid;
510
+	u64	vsid;
511
+};
512
+
498513 extern void slb_initialize(void);
499
-extern void slb_flush_and_rebolt(void);
514
+void slb_flush_and_restore_bolted(void);
500515 void slb_flush_all_realmode(void);
501516 void __slb_restore_bolted_realmode(void);
502517 void slb_restore_bolted_realmode(void);
518
+void slb_save_contents(struct slb_entry *slb_ptr);
519
+void slb_dump_contents(struct slb_entry *slb_ptr);
503520 
504521 extern void slb_vmalloc_update(void);
505522 extern void slb_set_size(u16 size);
....@@ -512,13 +529,9 @@
512529  * from mmu context id and effective segment id of the address.
513530  *
514531  * For user processes max context id is limited to MAX_USER_CONTEXT.
515
-
516
- * For kernel space, we use context ids 1-4 to map addresses as below:
517
- * NOTE: each context only support 64TB now.
518
- * 0x00001 -  [ 0xc000000000000000 - 0xc0003fffffffffff ]
519
- * 0x00002 -  [ 0xd000000000000000 - 0xd0003fffffffffff ]
520
- * 0x00003 -  [ 0xe000000000000000 - 0xe0003fffffffffff ]
521
- * 0x00004 -  [ 0xf000000000000000 - 0xf0003fffffffffff ]
532
+ * more details in get_user_context
533
+ *
534
+ * For kernel space get_kernel_context
522535  *
523536  * The proto-VSIDs are then scrambled into real VSIDs with the
524537  * multiplicative hash:
....@@ -559,6 +572,22 @@
559572 #define ESID_BITS_1T_MASK	((1 << ESID_BITS_1T) - 1)
560573 
561574 /*
575
+ * Now certain config support MAX_PHYSMEM more than 512TB. Hence we will need
576
+ * to use more than one context for linear mapping the kernel.
577
+ * For vmalloc and memmap, we use just one context with 512TB. With 64 byte
578
+ * struct page size, we need ony 32 TB in memmap for 2PB (51 bits (MAX_PHYSMEM_BITS)).
579
+ */
580
+#if (H_MAX_PHYSMEM_BITS > MAX_EA_BITS_PER_CONTEXT)
581
+#define MAX_KERNEL_CTX_CNT	(1UL << (H_MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT))
582
+#else
583
+#define MAX_KERNEL_CTX_CNT	1
584
+#endif
585
+
586
+#define MAX_VMALLOC_CTX_CNT	1
587
+#define MAX_IO_CTX_CNT		1
588
+#define MAX_VMEMMAP_CTX_CNT	1
589
+
590
+/*
562591  * 256MB segment
563592  * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments
564593  * available for user + kernel mapping. VSID 0 is reserved as invalid, contexts
....@@ -568,12 +597,13 @@
568597  * We also need to avoid the last segment of the last context, because that
569598  * would give a protovsid of 0x1fffffffff. That will result in a VSID 0
570599  * because of the modulo operation in vsid scramble.
600
+ *
571601  */
572602 #define MAX_USER_CONTEXT	((ASM_CONST(1) << CONTEXT_BITS) - 2)
573
-#define MIN_USER_CONTEXT	(5)
574603 
575
-/* Would be nice to use KERNEL_REGION_ID here */
576
-#define KERNEL_REGION_CONTEXT_OFFSET	(0xc - 1)
604
+// The + 2 accounts for INVALID_REGION and 1 more to avoid overlap with kernel
605
+#define MIN_USER_CONTEXT	(MAX_KERNEL_CTX_CNT + MAX_VMALLOC_CTX_CNT + \
606
+				 MAX_IO_CTX_CNT + MAX_VMEMMAP_CTX_CNT + 2)
577607 
578608 /*
579609  * For platforms that support on 65bit VA we limit the context bits
....@@ -624,8 +654,8 @@
624654 
625655 /* 4 bits per slice and we have one slice per 1TB */
626656 #define SLICE_ARRAY_SIZE	(H_PGTABLE_RANGE >> 41)
627
-#define TASK_SLICE_ARRAY_SZ(x)	((x)->context.slb_addr_limit >> 41)
628
-
657
+#define LOW_SLICE_ARRAY_SZ	(BITS_PER_LONG / BITS_PER_BYTE)
658
+#define TASK_SLICE_ARRAY_SZ(x)	((x)->hash_context->slb_addr_limit >> 41)
629659 #ifndef __ASSEMBLY__
630660 
631661 #ifdef CONFIG_PPC_SUBPAGE_PROT
....@@ -654,11 +684,40 @@
654684 #define SBP_L3_SHIFT		(SBP_L2_SHIFT + SBP_L2_BITS)
655685 
656686 extern void subpage_prot_free(struct mm_struct *mm);
657
-extern void subpage_prot_init_new_context(struct mm_struct *mm);
658687 #else
659688 static inline void subpage_prot_free(struct mm_struct *mm) {}
660
-static inline void subpage_prot_init_new_context(struct mm_struct *mm) { }
661689 #endif /* CONFIG_PPC_SUBPAGE_PROT */
690
+
691
+/*
692
+ * One bit per slice. We have lower slices which cover 256MB segments
693
+ * upto 4G range. That gets us 16 low slices. For the rest we track slices
694
+ * in 1TB size.
695
+ */
696
+struct slice_mask {
697
+	u64 low_slices;
698
+	DECLARE_BITMAP(high_slices, SLICE_NUM_HIGH);
699
+};
700
+
701
+struct hash_mm_context {
702
+	u16 user_psize; /* page size index */
703
+
704
+	/* SLB page size encodings*/
705
+	unsigned char low_slices_psize[LOW_SLICE_ARRAY_SZ];
706
+	unsigned char high_slices_psize[SLICE_ARRAY_SIZE];
707
+	unsigned long slb_addr_limit;
708
+#ifdef CONFIG_PPC_64K_PAGES
709
+	struct slice_mask mask_64k;
710
+#endif
711
+	struct slice_mask mask_4k;
712
+#ifdef CONFIG_HUGETLB_PAGE
713
+	struct slice_mask mask_16m;
714
+	struct slice_mask mask_16g;
715
+#endif
716
+
717
+#ifdef CONFIG_PPC_SUBPAGE_PROT
718
+	struct subpage_prot_table *spt;
719
+#endif /* CONFIG_PPC_SUBPAGE_PROT */
720
+};
662721 
663722 #if 0
664723 /*
....@@ -714,7 +773,7 @@
714773 	/*
715774 	 * Bad address. We return VSID 0 for that
716775 	 */
717
-	if ((ea & ~REGION_MASK) >= H_PGTABLE_RANGE)
776
+	if ((ea & EA_MASK)  >= H_PGTABLE_RANGE)
718777 		return 0;
719778 
720779 	if (!mmu_has_feature(MMU_FTR_68_BIT_VA))
....@@ -734,6 +793,40 @@
734793 }
735794 
736795 /*
796
+ * For kernel space, we use context ids as
797
+ * below. Range is 512TB per context.
798
+ *
799
+ * 0x00001 -  [ 0xc000000000000000 - 0xc001ffffffffffff]
800
+ * 0x00002 -  [ 0xc002000000000000 - 0xc003ffffffffffff]
801
+ * 0x00003 -  [ 0xc004000000000000 - 0xc005ffffffffffff]
802
+ * 0x00004 -  [ 0xc006000000000000 - 0xc007ffffffffffff]
803
+ *
804
+ * vmap, IO, vmemap
805
+ *
806
+ * 0x00005 -  [ 0xc008000000000000 - 0xc009ffffffffffff]
807
+ * 0x00006 -  [ 0xc00a000000000000 - 0xc00bffffffffffff]
808
+ * 0x00007 -  [ 0xc00c000000000000 - 0xc00dffffffffffff]
809
+ *
810
+ */
811
+static inline unsigned long get_kernel_context(unsigned long ea)
812
+{
813
+	unsigned long region_id = get_region_id(ea);
814
+	unsigned long ctx;
815
+	/*
816
+	 * Depending on Kernel config, kernel region can have one context
817
+	 * or more.
818
+	 */
819
+	if (region_id == LINEAR_MAP_REGION_ID) {
820
+		/*
821
+		 * We already verified ea to be not beyond the addr limit.
822
+		 */
823
+		ctx =  1 + ((ea & EA_MASK) >> MAX_EA_BITS_PER_CONTEXT);
824
+	} else
825
+		ctx = region_id + MAX_KERNEL_CTX_CNT - 1;
826
+	return ctx;
827
+}
828
+
829
+/*
737830  * This is only valid for addresses >= PAGE_OFFSET
738831  */
739832 static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize)
....@@ -743,20 +836,7 @@
743836 	if (!is_kernel_addr(ea))
744837 		return 0;
745838 
746
-	/*
747
-	 * For kernel space, we use context ids 1-4 to map the address space as
748
-	 * below:
749
-	 *
750
-	 * 0x00001 -  [ 0xc000000000000000 - 0xc0003fffffffffff ]
751
-	 * 0x00002 -  [ 0xd000000000000000 - 0xd0003fffffffffff ]
752
-	 * 0x00003 -  [ 0xe000000000000000 - 0xe0003fffffffffff ]
753
-	 * 0x00004 -  [ 0xf000000000000000 - 0xf0003fffffffffff ]
754
-	 *
755
-	 * So we can compute the context from the region (top nibble) by
756
-	 * subtracting 11, or 0xc - 1.
757
-	 */
758
-	context = (ea >> 60) - KERNEL_REGION_CONTEXT_OFFSET;
759
-
839
+	context = get_kernel_context(ea);
760840 	return get_vsid(context, ea, ssize);
761841 }
762842