forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/arch/powerpc/include/asm/nohash/32/pgtable.h
....@@ -2,16 +2,12 @@
22 #ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H
33 #define _ASM_POWERPC_NOHASH_32_PGTABLE_H
44 
5
-#define __ARCH_USE_5LEVEL_HACK
65 #include <asm-generic/pgtable-nopmd.h>
76 
87 #ifndef __ASSEMBLY__
98 #include <linux/sched.h>
109 #include <linux/threads.h>
1110 #include <asm/mmu.h>			/* For sub-arch specific PPC_PIN_SIZE */
12
-#include <asm/asm-405.h>
13
-
14
-extern unsigned long ioremap_bot;
1511 
1612 #ifdef CONFIG_44x
1713 extern int icache_44x_need_flush;
....@@ -32,6 +28,8 @@
3228 #define PMD_TABLE_SIZE	0
3329 #define PUD_TABLE_SIZE	0
3430 #define PGD_TABLE_SIZE	(sizeof(pgd_t) << PGD_INDEX_SIZE)
31
+
32
+#define PMD_MASKED_BITS (PTE_TABLE_SIZE - 1)
3533 #endif	/* __ASSEMBLY__ */
3634 
3735 #define PTRS_PER_PTE	(1 << PTE_INDEX_SIZE)
....@@ -64,27 +62,35 @@
6462 #define pgd_ERROR(e) \
6563 	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
6664 
65
+#ifndef __ASSEMBLY__
66
+
67
+int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot);
68
+void unmap_kernel_page(unsigned long va);
69
+
70
+#endif /* !__ASSEMBLY__ */
71
+
72
+
6773 /*
6874  * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary
6975  * value (for now) on others, from where we can start layout kernel
7076  * virtual space that goes below PKMAP and FIXMAP
7177  */
72
-#ifdef CONFIG_HIGHMEM
73
-#define KVIRT_TOP	PKMAP_BASE
74
-#else
75
-#define KVIRT_TOP	(0xfe000000UL)	/* for now, could be FIXMAP_BASE ? */
76
-#endif
78
+#include <asm/fixmap.h>
7779 
7880 /*
7981  * ioremap_bot starts at that address. Early ioremaps move down from there,
8082  * until mem_init() at which point this becomes the top of the vmalloc
8183  * and ioremap space
8284  */
83
-#ifdef CONFIG_NOT_COHERENT_CACHE
84
-#define IOREMAP_TOP	((KVIRT_TOP - CONFIG_CONSISTENT_SIZE) & PAGE_MASK)
85
+#ifdef CONFIG_HIGHMEM
86
+#define IOREMAP_TOP	PKMAP_BASE
8587 #else
86
-#define IOREMAP_TOP	KVIRT_TOP
88
+#define IOREMAP_TOP	FIXADDR_START
8789 #endif
90
+
91
+/* PPC32 shares vmalloc area with ioremap */
92
+#define IOREMAP_START	VMALLOC_START
93
+#define IOREMAP_END	VMALLOC_END
8894 
8995 /*
9096  * Just any arbitrary offset to the start of the vmalloc VM area: the
....@@ -105,11 +111,16 @@
105111  */
106112 #define VMALLOC_OFFSET (0x1000000) /* 16M */
107113 #ifdef PPC_PIN_SIZE
108
-#define VMALLOC_START (((_ALIGN((long)high_memory, PPC_PIN_SIZE) + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)))
114
+#define VMALLOC_START (((ALIGN((long)high_memory, PPC_PIN_SIZE) + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)))
109115 #else
110116 #define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)))
111117 #endif
118
+
119
+#ifdef CONFIG_KASAN_VMALLOC
120
+#define VMALLOC_END	ALIGN_DOWN(ioremap_bot, PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)
121
+#else
112122 #define VMALLOC_END	ioremap_bot
123
+#endif
113124 
114125 /*
115126  * Bits in a linux-style PTE.  These match the bits in the
....@@ -128,13 +139,66 @@
128139 #include <asm/nohash/32/pte-8xx.h>
129140 #endif
130141 
131
-/* And here we include common definitions */
132
-#include <asm/pte-common.h>
142
+/*
143
+ * Location of the PFN in the PTE. Most 32-bit platforms use the same
144
+ * as _PAGE_SHIFT here (ie, naturally aligned).
145
+ * Platform who don't just pre-define the value so we don't override it here.
146
+ */
147
+#ifndef PTE_RPN_SHIFT
148
+#define PTE_RPN_SHIFT	(PAGE_SHIFT)
149
+#endif
150
+
151
+/*
152
+ * The mask covered by the RPN must be a ULL on 32-bit platforms with
153
+ * 64-bit PTEs.
154
+ */
155
+#if defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT)
156
+#define PTE_RPN_MASK	(~((1ULL << PTE_RPN_SHIFT) - 1))
157
+#define MAX_POSSIBLE_PHYSMEM_BITS 36
158
+#else
159
+#define PTE_RPN_MASK	(~((1UL << PTE_RPN_SHIFT) - 1))
160
+#define MAX_POSSIBLE_PHYSMEM_BITS 32
161
+#endif
162
+
163
+/*
164
+ * _PAGE_CHG_MASK masks of bits that are to be preserved across
165
+ * pgprot changes.
166
+ */
167
+#define _PAGE_CHG_MASK	(PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL)
133168 
134169 #ifndef __ASSEMBLY__
135170 
136171 #define pte_clear(mm, addr, ptep) \
137
-	do { pte_update(ptep, ~0, 0); } while (0)
172
+	do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0)
173
+
174
+#ifndef pte_mkwrite
175
+static inline pte_t pte_mkwrite(pte_t pte)
176
+{
177
+	return __pte(pte_val(pte) | _PAGE_RW);
178
+}
179
+#endif
180
+
181
+static inline pte_t pte_mkdirty(pte_t pte)
182
+{
183
+	return __pte(pte_val(pte) | _PAGE_DIRTY);
184
+}
185
+
186
+static inline pte_t pte_mkyoung(pte_t pte)
187
+{
188
+	return __pte(pte_val(pte) | _PAGE_ACCESSED);
189
+}
190
+
191
+#ifndef pte_wrprotect
192
+static inline pte_t pte_wrprotect(pte_t pte)
193
+{
194
+	return __pte(pte_val(pte) & ~_PAGE_RW);
195
+}
196
+#endif
197
+
198
+static inline pte_t pte_mkexec(pte_t pte)
199
+{
200
+	return __pte(pte_val(pte) | _PAGE_EXEC);
201
+}
138202 
139203 #define pmd_none(pmd)		(!pmd_val(pmd))
140204 #define	pmd_bad(pmd)		(pmd_val(pmd) & _PMD_BAD)
....@@ -143,8 +207,6 @@
143207 {
144208 	*pmdp = __pmd(0);
145209 }
146
-
147
-
148210 
149211 /*
150212  * PTE updates. This function is called whenever an existing
....@@ -160,29 +222,64 @@
160222  * that an executable user mapping was modified, which is needed
161223  * to properly flush the virtually tagged instruction cache of
162224  * those implementations.
225
+ *
226
+ * On the 8xx, the page tables are a bit special. For 16k pages, we have
227
+ * 4 identical entries. For 512k pages, we have 128 entries as if it was
228
+ * 4k pages, but they are flagged as 512k pages for the hardware.
229
+ * For other page sizes, we have a single entry in the table.
163230  */
164
-#ifndef CONFIG_PTE_64BIT
165
-static inline unsigned long pte_update(pte_t *p,
166
-				       unsigned long clr,
167
-				       unsigned long set)
168
-{
169
-#ifdef PTE_ATOMIC_UPDATES
170
-	unsigned long old, tmp;
231
+#ifdef CONFIG_PPC_8xx
232
+static pmd_t *pmd_off(struct mm_struct *mm, unsigned long addr);
233
+static int hugepd_ok(hugepd_t hpd);
171234 
172
-	__asm__ __volatile__("\
173
-1:	lwarx	%0,0,%3\n\
174
-	andc	%1,%0,%4\n\
175
-	or	%1,%1,%5\n"
176
-	PPC405_ERR77(0,%3)
177
-"	stwcx.	%1,0,%3\n\
178
-	bne-	1b"
179
-	: "=&r" (old), "=&r" (tmp), "=m" (*p)
180
-	: "r" (p), "r" (clr), "r" (set), "m" (*p)
181
-	: "cc" );
182
-#else /* PTE_ATOMIC_UPDATES */
183
-	unsigned long old = pte_val(*p);
184
-	*p = __pte((old & ~clr) | set);
185
-#endif /* !PTE_ATOMIC_UPDATES */
235
+static int number_of_cells_per_pte(pmd_t *pmd, pte_basic_t val, int huge)
236
+{
237
+	if (!huge)
238
+		return PAGE_SIZE / SZ_4K;
239
+	else if (hugepd_ok(*((hugepd_t *)pmd)))
240
+		return 1;
241
+	else if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !(val & _PAGE_HUGE))
242
+		return SZ_16K / SZ_4K;
243
+	else
244
+		return SZ_512K / SZ_4K;
245
+}
246
+
247
+static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,
248
+				     unsigned long clr, unsigned long set, int huge)
249
+{
250
+	pte_basic_t *entry = &p->pte;
251
+	pte_basic_t old = pte_val(*p);
252
+	pte_basic_t new = (old & ~(pte_basic_t)clr) | set;
253
+	int num, i;
254
+	pmd_t *pmd = pmd_off(mm, addr);
255
+
256
+	num = number_of_cells_per_pte(pmd, new, huge);
257
+
258
+	for (i = 0; i < num; i++, entry++, new += SZ_4K)
259
+		*entry = new;
260
+
261
+	return old;
262
+}
263
+
264
+#ifdef CONFIG_PPC_16K_PAGES
265
+#define __HAVE_ARCH_PTEP_GET
266
+static inline pte_t ptep_get(pte_t *ptep)
267
+{
268
+	pte_basic_t val = READ_ONCE(ptep->pte);
269
+	pte_t pte = {val, val, val, val};
270
+
271
+	return pte;
272
+}
273
+#endif /* CONFIG_PPC_16K_PAGES */
274
+
275
+#else
276
+static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p,
277
+				     unsigned long clr, unsigned long set, int huge)
278
+{
279
+	pte_basic_t old = pte_val(*p);
280
+	pte_basic_t new = (old & ~(pte_basic_t)clr) | set;
281
+
282
+	*p = __pte(new);
186283 
187284 #ifdef CONFIG_44x
188285 	if ((old & _PAGE_USER) && (old & _PAGE_EXEC))
....@@ -190,79 +287,48 @@
190287 #endif
191288 	return old;
192289 }
193
-#else /* CONFIG_PTE_64BIT */
194
-static inline unsigned long long pte_update(pte_t *p,
195
-					    unsigned long clr,
196
-					    unsigned long set)
197
-{
198
-#ifdef PTE_ATOMIC_UPDATES
199
-	unsigned long long old;
200
-	unsigned long tmp;
201
-
202
-	__asm__ __volatile__("\
203
-1:	lwarx	%L0,0,%4\n\
204
-	lwzx	%0,0,%3\n\
205
-	andc	%1,%L0,%5\n\
206
-	or	%1,%1,%6\n"
207
-	PPC405_ERR77(0,%3)
208
-"	stwcx.	%1,0,%4\n\
209
-	bne-	1b"
210
-	: "=&r" (old), "=&r" (tmp), "=m" (*p)
211
-	: "r" (p), "r" ((unsigned long)(p) + 4), "r" (clr), "r" (set), "m" (*p)
212
-	: "cc" );
213
-#else /* PTE_ATOMIC_UPDATES */
214
-	unsigned long long old = pte_val(*p);
215
-	*p = __pte((old & ~(unsigned long long)clr) | set);
216
-#endif /* !PTE_ATOMIC_UPDATES */
217
-
218
-#ifdef CONFIG_44x
219
-	if ((old & _PAGE_USER) && (old & _PAGE_EXEC))
220
-		icache_44x_need_flush = 1;
221290 #endif
222
-	return old;
223
-}
224
-#endif /* CONFIG_PTE_64BIT */
225291 
226292 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
227
-static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep)
293
+static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
294
+					      unsigned long addr, pte_t *ptep)
228295 {
229296 	unsigned long old;
230
-	old = pte_update(ptep, _PAGE_ACCESSED, 0);
297
+	old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
231298 	return (old & _PAGE_ACCESSED) != 0;
232299 }
233300 #define ptep_test_and_clear_young(__vma, __addr, __ptep) \
234
-	__ptep_test_and_clear_young((__vma)->vm_mm->context.id, __addr, __ptep)
301
+	__ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep)
235302 
236303 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
237304 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
238305 				       pte_t *ptep)
239306 {
240
-	return __pte(pte_update(ptep, ~0, 0));
307
+	return __pte(pte_update(mm, addr, ptep, ~0, 0, 0));
241308 }
242309 
243310 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
244311 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
245312 				      pte_t *ptep)
246313 {
247
-	pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), _PAGE_RO);
248
-}
249
-static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
250
-					   unsigned long addr, pte_t *ptep)
251
-{
252
-	ptep_set_wrprotect(mm, addr, ptep);
253
-}
314
+	unsigned long clr = ~pte_val(pte_wrprotect(__pte(~0)));
315
+	unsigned long set = pte_val(pte_wrprotect(__pte(0)));
254316 
317
+	pte_update(mm, addr, ptep, clr, set, 0);
318
+}
255319 
256320 static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
257321 					   pte_t *ptep, pte_t entry,
258322 					   unsigned long address,
259323 					   int psize)
260324 {
261
-	unsigned long set = pte_val(entry) &
262
-		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
263
-	unsigned long clr = ~pte_val(entry) & (_PAGE_RO | _PAGE_NA);
325
+	pte_t pte_set = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte_mkexec(__pte(0)))));
326
+	pte_t pte_clr = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte_mkexec(__pte(~0)))));
327
+	unsigned long set = pte_val(entry) & pte_val(pte_set);
328
+	unsigned long clr = ~pte_val(entry) & ~pte_val(pte_clr);
329
+	int huge = psize > mmu_virtual_psize ? 1 : 0;
264330 
265
-	pte_update(ptep, clr, set);
331
+	pte_update(vma->vm_mm, address, ptep, clr, set, huge);
266332 
267333 	flush_tlb_page(vma, address);
268334 }
....@@ -283,33 +349,14 @@
283349  * of the pte page.  -- paulus
284350  */
285351 #ifndef CONFIG_BOOKE
286
-#define pmd_page_vaddr(pmd)	\
287
-	((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
288352 #define pmd_page(pmd)		\
289353 	pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
290354 #else
291355 #define pmd_page_vaddr(pmd)	\
292
-	((unsigned long) (pmd_val(pmd) & PAGE_MASK))
356
+	((unsigned long)(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1)))
293357 #define pmd_page(pmd)		\
294358 	pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT))
295359 #endif
296
-
297
-/* to find an entry in a kernel page-table-directory */
298
-#define pgd_offset_k(address) pgd_offset(&init_mm, address)
299
-
300
-/* to find an entry in a page-table-directory */
301
-#define pgd_index(address)	 ((address) >> PGDIR_SHIFT)
302
-#define pgd_offset(mm, address)	 ((mm)->pgd + pgd_index(address))
303
-
304
-/* Find an entry in the third-level page table.. */
305
-#define pte_index(address)		\
306
-	(((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
307
-#define pte_offset_kernel(dir, addr)	\
308
-	(pmd_bad(*(dir)) ? NULL : (pte_t *)pmd_page_vaddr(*(dir)) + \
309
-				  pte_index(addr))
310
-#define pte_offset_map(dir, addr)		\
311
-	((pte_t *) kmap_atomic(pmd_page(*(dir))) + pte_index(addr))
312
-#define pte_unmap(pte)		kunmap_atomic(pte)
313360 
314361 /*
315362  * Encode and decode a swap entry.
....@@ -322,8 +369,6 @@
322369 #define __swp_entry(type, offset)	((swp_entry_t) { (type) | ((offset) << 5) })
323370 #define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> 3 })
324371 #define __swp_entry_to_pte(x)		((pte_t) { (x).val << 3 })
325
-
326
-int map_kernel_page(unsigned long va, phys_addr_t pa, int flags);
327372 
328373 #endif /* !__ASSEMBLY__ */
329374