blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/arch/arm/mm/mmu.c
....@@ -1,11 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  *  linux/arch/arm/mm/mmu.c
34  *
45  *  Copyright (C) 1995-2005 Russell King
5
- *
6
- * This program is free software; you can redistribute it and/or modify
7
- * it under the terms of the GNU General Public License version 2 as
8
- * published by the Free Software Foundation.
96  */
107 #include <linux/module.h>
118 #include <linux/kernel.h>
....@@ -20,7 +17,6 @@
2017 
2118 #include <asm/cp15.h>
2219 #include <asm/cputype.h>
23
-#include <asm/sections.h>
2420 #include <asm/cachetype.h>
2521 #include <asm/fixmap.h>
2622 #include <asm/sections.h>
....@@ -32,6 +28,7 @@
3228 #include <asm/traps.h>
3329 #include <asm/procinfo.h>
3430 #include <asm/memory.h>
31
+#include <asm/pgalloc.h>
3532 
3633 #include <asm/mach/arch.h>
3734 #include <asm/mach/map.h>
....@@ -41,6 +38,8 @@
4138 #include "fault.h"
4239 #include "mm.h"
4340 #include "tcm.h"
41
+
42
+extern unsigned long __atags_pointer;
4443 
4544 /*
4645  * empty_zero_page is a special page that is used for
....@@ -66,9 +65,6 @@
6665 static unsigned int ecc_mask __initdata = 0;
6766 pgprot_t pgprot_user;
6867 pgprot_t pgprot_kernel;
69
-pgprot_t pgprot_hyp_device;
70
-pgprot_t pgprot_s2;
71
-pgprot_t pgprot_s2_device;
7268 
7369 EXPORT_SYMBOL(pgprot_user);
7470 EXPORT_SYMBOL(pgprot_kernel);
....@@ -78,16 +74,7 @@
7874 	unsigned int	cr_mask;
7975 	pmdval_t	pmd;
8076 	pteval_t	pte;
81
-	pteval_t	pte_s2;
8277 };
83
-
84
-#ifdef CONFIG_ARM_LPAE
85
-#define s2_policy(policy)	policy
86
-#else
87
-#define s2_policy(policy)	0
88
-#endif
89
-
90
-unsigned long kimage_voffset __ro_after_init;
9178 
9279 static struct cachepolicy cache_policies[] __initdata = {
9380 	{
....@@ -95,31 +82,26 @@
9582 		.cr_mask	= CR_W|CR_C,
9683 		.pmd		= PMD_SECT_UNCACHED,
9784 		.pte		= L_PTE_MT_UNCACHED,
98
-		.pte_s2		= s2_policy(L_PTE_S2_MT_UNCACHED),
9985 	}, {
10086 		.policy		= "buffered",
10187 		.cr_mask	= CR_C,
10288 		.pmd		= PMD_SECT_BUFFERED,
10389 		.pte		= L_PTE_MT_BUFFERABLE,
104
-		.pte_s2		= s2_policy(L_PTE_S2_MT_UNCACHED),
10590 	}, {
10691 		.policy		= "writethrough",
10792 		.cr_mask	= 0,
10893 		.pmd		= PMD_SECT_WT,
10994 		.pte		= L_PTE_MT_WRITETHROUGH,
110
-		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITETHROUGH),
11195 	}, {
11296 		.policy		= "writeback",
11397 		.cr_mask	= 0,
11498 		.pmd		= PMD_SECT_WB,
11599 		.pte		= L_PTE_MT_WRITEBACK,
116
-		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITEBACK),
117100 	}, {
118101 		.policy		= "writealloc",
119102 		.cr_mask	= 0,
120103 		.pmd		= PMD_SECT_WBWA,
121104 		.pte		= L_PTE_MT_WRITEALLOC,
122
-		.pte_s2		= s2_policy(L_PTE_S2_MT_WRITEBACK),
123105 	}
124106 };
125107 
....@@ -230,12 +212,14 @@
230212 static int __init early_cachepolicy(char *p)
231213 {
232214 	pr_warn("cachepolicy kernel parameter not supported without cp15\n");
215
+	return 0;
233216 }
234217 early_param("cachepolicy", early_cachepolicy);
235218 
236219 static int __init noalign_setup(char *__unused)
237220 {
238221 	pr_warn("noalign kernel parameter not supported without cp15\n");
222
+	return 1;
239223 }
240224 __setup("noalign", noalign_setup);
241225 
....@@ -249,9 +233,6 @@
249233 	[MT_DEVICE] = {		  /* Strongly ordered / ARMv6 shared device */
250234 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
251235 				  L_PTE_SHARED,
252
-		.prot_pte_s2	= s2_policy(PROT_PTE_S2_DEVICE) |
253
-				  s2_policy(L_PTE_S2_MT_DEV_SHARED) |
254
-				  L_PTE_SHARED,
255236 		.prot_l1	= PMD_TYPE_TABLE,
256237 		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_S,
257238 		.domain		= DOMAIN_IO,
....@@ -262,7 +243,7 @@
262243 		.prot_sect	= PROT_SECT_DEVICE,
263244 		.domain		= DOMAIN_IO,
264245 	},
265
-	[MT_DEVICE_CACHED] = {	  /* ioremap_cached */
246
+	[MT_DEVICE_CACHED] = {	  /* ioremap_cache */
266247 		.prot_pte	= PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
267248 		.prot_l1	= PMD_TYPE_TABLE,
268249 		.prot_sect	= PROT_SECT_DEVICE | PMD_SECT_WB,
....@@ -313,6 +294,17 @@
313294 			     L_PTE_XN,
314295 		.prot_l1   = PMD_TYPE_TABLE,
315296 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
297
+		.domain    = DOMAIN_KERNEL,
298
+	},
299
+	[MT_MEMORY_RO] = {
300
+		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
301
+			     L_PTE_XN | L_PTE_RDONLY,
302
+		.prot_l1   = PMD_TYPE_TABLE,
303
+#ifdef CONFIG_ARM_LPAE
304
+		.prot_sect = PMD_TYPE_SECT | L_PMD_SECT_RDONLY | PMD_SECT_AP2,
305
+#else
306
+		.prot_sect = PMD_TYPE_SECT,
307
+#endif
316308 		.domain    = DOMAIN_KERNEL,
317309 	},
318310 	[MT_ROM] = {
....@@ -377,11 +369,7 @@
377369 
378370 static inline pmd_t * __init fixmap_pmd(unsigned long addr)
379371 {
380
-	pgd_t *pgd = pgd_offset_k(addr);
381
-	pud_t *pud = pud_offset(pgd, addr);
382
-	pmd_t *pmd = pmd_offset(pud, addr);
383
-
384
-	return pmd;
372
+	return pmd_off_k(addr);
385373 }
386374 
387375 void __init early_fixmap_init(void)
....@@ -437,7 +425,6 @@
437425 	struct cachepolicy *cp;
438426 	unsigned int cr = get_cr();
439427 	pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
440
-	pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
441428 	int cpu_arch = cpu_architecture();
442429 	int i;
443430 
....@@ -514,6 +501,7 @@
514501 
515502 			/* Also setup NX memory mapping */
516503 			mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_XN;
504
+			mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_XN;
517505 		}
518506 		if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
519507 			/*
....@@ -561,9 +549,6 @@
561549 	 */
562550 	cp = &cache_policies[cachepolicy];
563551 	vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
564
-	s2_pgprot = cp->pte_s2;
565
-	hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte;
566
-	s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2;
567552 
568553 #ifndef CONFIG_ARM_LPAE
569554 	/*
....@@ -596,6 +581,7 @@
596581 		mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
597582 		mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
598583 		mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
584
+		mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
599585 #endif
600586 
601587 		/*
....@@ -607,7 +593,6 @@
607593 			user_pgprot |= L_PTE_SHARED;
608594 			kern_pgprot |= L_PTE_SHARED;
609595 			vecs_pgprot |= L_PTE_SHARED;
610
-			s2_pgprot |= L_PTE_SHARED;
611596 			mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
612597 			mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
613598 			mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
....@@ -616,6 +601,8 @@
616601 			mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
617602 			mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S;
618603 			mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED;
604
+			mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_S;
605
+			mem_types[MT_MEMORY_RO].prot_pte |= L_PTE_SHARED;
619606 			mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
620607 			mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S;
621608 			mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED;
....@@ -669,9 +656,6 @@
669656 	pgprot_user   = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
670657 	pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
671658 				 L_PTE_DIRTY | kern_pgprot);
672
-	pgprot_s2  = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
673
-	pgprot_s2_device  = __pgprot(s2_device_pgprot);
674
-	pgprot_hyp_device  = __pgprot(hyp_device_pgprot);
675659 
676660 	mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
677661 	mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
....@@ -679,6 +663,8 @@
679663 	mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot;
680664 	mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;
681665 	mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot;
666
+	mem_types[MT_MEMORY_RO].prot_sect |= ecc_mask | cp->pmd;
667
+	mem_types[MT_MEMORY_RO].prot_pte |= kern_pgprot;
682668 	mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
683669 	mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask;
684670 	mem_types[MT_ROM].prot_sect |= cp->pmd;
....@@ -719,23 +705,22 @@
719705 
720706 #define vectors_base()	(vectors_high() ? 0xffff0000 : 0)
721707 
722
-static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
723
-{
724
-	void *ptr = __va(memblock_alloc(sz, align));
725
-	memset(ptr, 0, sz);
726
-	return ptr;
727
-}
728
-
729708 static void __init *early_alloc(unsigned long sz)
730709 {
731
-	return early_alloc_aligned(sz, sz);
710
+	void *ptr = memblock_alloc(sz, sz);
711
+
712
+	if (!ptr)
713
+		panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
714
+		      __func__, sz, sz);
715
+
716
+	return ptr;
732717 }
733718 
734719 static void *__init late_alloc(unsigned long sz)
735720 {
736
-	void *ptr = (void *)__get_free_pages(PGALLOC_GFP, get_order(sz));
721
+	void *ptr = (void *)__get_free_pages(GFP_PGTABLE_KERNEL, get_order(sz));
737722 
738
-	if (!ptr || !pgtable_page_ctor(virt_to_page(ptr)))
723
+	if (!ptr || !pgtable_pte_page_ctor(virt_to_page(ptr)))
739724 		BUG();
740725 	return ptr;
741726 }
....@@ -831,12 +816,12 @@
831816 	} while (pmd++, addr = next, addr != end);
832817 }
833818 
834
-static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
819
+static void __init alloc_init_pud(p4d_t *p4d, unsigned long addr,
835820 				  unsigned long end, phys_addr_t phys,
836821 				  const struct mem_type *type,
837822 				  void *(*alloc)(unsigned long sz), bool ng)
838823 {
839
-	pud_t *pud = pud_offset(pgd, addr);
824
+	pud_t *pud = pud_offset(p4d, addr);
840825 	unsigned long next;
841826 
842827 	do {
....@@ -844,6 +829,21 @@
844829 		alloc_init_pmd(pud, addr, next, phys, type, alloc, ng);
845830 		phys += next - addr;
846831 	} while (pud++, addr = next, addr != end);
832
+}
833
+
834
+static void __init alloc_init_p4d(pgd_t *pgd, unsigned long addr,
835
+				  unsigned long end, phys_addr_t phys,
836
+				  const struct mem_type *type,
837
+				  void *(*alloc)(unsigned long sz), bool ng)
838
+{
839
+	p4d_t *p4d = p4d_offset(pgd, addr);
840
+	unsigned long next;
841
+
842
+	do {
843
+		next = p4d_addr_end(addr, end);
844
+		alloc_init_pud(p4d, addr, next, phys, type, alloc, ng);
845
+		phys += next - addr;
846
+	} while (p4d++, addr = next, addr != end);
847847 }
848848 
849849 #ifndef CONFIG_ARM_LPAE
....@@ -893,7 +893,8 @@
893893 	pgd = pgd_offset(mm, addr);
894894 	end = addr + length;
895895 	do {
896
-		pud_t *pud = pud_offset(pgd, addr);
896
+		p4d_t *p4d = p4d_offset(pgd, addr);
897
+		pud_t *pud = pud_offset(p4d, addr);
897898 		pmd_t *pmd = pmd_offset(pud, addr);
898899 		int i;
899900 
....@@ -944,7 +945,7 @@
944945 	do {
945946 		unsigned long next = pgd_addr_end(addr, end);
946947 
947
-		alloc_init_pud(pgd, addr, next, phys, type, alloc, ng);
948
+		alloc_init_p4d(pgd, addr, next, phys, type, alloc, ng);
948949 
949950 		phys += next - addr;
950951 		addr = next;
....@@ -966,7 +967,7 @@
966967 		return;
967968 	}
968969 
969
-	if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
970
+	if (md->type == MT_DEVICE &&
970971 	    md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START &&
971972 	    (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
972973 		pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n",
....@@ -980,7 +981,13 @@
980981 				bool ng)
981982 {
982983 #ifdef CONFIG_ARM_LPAE
983
-	pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
984
+	p4d_t *p4d;
985
+	pud_t *pud;
986
+
987
+	p4d = p4d_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
988
+	if (WARN_ON(!p4d))
989
+		return;
990
+	pud = pud_alloc(mm, p4d, md->virtual);
984991 	if (WARN_ON(!pud))
985992 		return;
986993 	pmd_alloc(mm, pud, 0);
....@@ -1000,7 +1007,10 @@
10001007 	if (!nr)
10011008 		return;
10021009 
1003
-	svm = early_alloc_aligned(sizeof(*svm) * nr, __alignof__(*svm));
1010
+	svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm));
1011
+	if (!svm)
1012
+		panic("%s: Failed to allocate %zu bytes align=0x%zx\n",
1013
+		      __func__, sizeof(*svm) * nr, __alignof__(*svm));
10041014 
10051015 	for (md = io_desc; nr; md++, nr--) {
10061016 		create_mapping(md);
....@@ -1022,7 +1032,10 @@
10221032 	struct vm_struct *vm;
10231033 	struct static_vm *svm;
10241034 
1025
-	svm = early_alloc_aligned(sizeof(*svm), __alignof__(*svm));
1035
+	svm = memblock_alloc(sizeof(*svm), __alignof__(*svm));
1036
+	if (!svm)
1037
+		panic("%s: Failed to allocate %zu bytes align=0x%zx\n",
1038
+		      __func__, sizeof(*svm), __alignof__(*svm));
10261039 
10271040 	vm = &svm->vm;
10281041 	vm->addr = (void *)addr;
....@@ -1161,9 +1174,8 @@
11611174 
11621175 void __init adjust_lowmem_bounds(void)
11631176 {
1164
-	phys_addr_t memblock_limit = 0;
1165
-	u64 vmalloc_limit;
1166
-	struct memblock_region *reg;
1177
+	phys_addr_t block_start, block_end, memblock_limit = 0;
1178
+	u64 vmalloc_limit, i;
11671179 	phys_addr_t lowmem_limit = 0;
11681180 
11691181 	/*
....@@ -1179,26 +1191,18 @@
11791191 	 * The first usable region must be PMD aligned. Mark its start
11801192 	 * as MEMBLOCK_NOMAP if it isn't
11811193 	 */
1182
-	for_each_memblock(memory, reg) {
1183
-		if (!memblock_is_nomap(reg)) {
1184
-			if (!IS_ALIGNED(reg->base, PMD_SIZE)) {
1185
-				phys_addr_t len;
1194
+	for_each_mem_range(i, &block_start, &block_end) {
1195
+		if (!IS_ALIGNED(block_start, PMD_SIZE)) {
1196
+			phys_addr_t len;
11861197 
1187
-				len = round_up(reg->base, PMD_SIZE) - reg->base;
1188
-				memblock_mark_nomap(reg->base, len);
1189
-			}
1190
-			break;
1198
+			len = round_up(block_start, PMD_SIZE) - block_start;
1199
+			memblock_mark_nomap(block_start, len);
11911200 		}
1201
+		break;
11921202 	}
11931203 
1194
-	for_each_memblock(memory, reg) {
1195
-		phys_addr_t block_start = reg->base;
1196
-		phys_addr_t block_end = reg->base + reg->size;
1197
-
1198
-		if (memblock_is_nomap(reg))
1199
-			continue;
1200
-
1201
-		if (reg->base < vmalloc_limit) {
1204
+	for_each_mem_range(i, &block_start, &block_end) {
1205
+		if (block_start < vmalloc_limit) {
12021206 			if (block_end > lowmem_limit)
12031207 				/*
12041208 				 * Compare as u64 to ensure vmalloc_limit does
....@@ -1350,6 +1354,15 @@
13501354 	for (addr = VMALLOC_START; addr < (FIXADDR_TOP & PMD_MASK); addr += PMD_SIZE)
13511355 		pmd_clear(pmd_off_k(addr));
13521356 
1357
+	if (__atags_pointer) {
1358
+		/* create a read-only mapping of the device tree */
1359
+		map.pfn = __phys_to_pfn(__atags_pointer & SECTION_MASK);
1360
+		map.virtual = FDT_FIXED_BASE;
1361
+		map.length = FDT_FIXED_SIZE;
1362
+		map.type = MT_MEMORY_RO;
1363
+		create_mapping(&map);
1364
+	}
1365
+
13531366 	/*
13541367 	 * Map the kernel if it is XIP.
13551368 	 * It is always first in the modulearea.
....@@ -1447,18 +1460,14 @@
14471460 
14481461 static void __init map_lowmem(void)
14491462 {
1450
-	struct memblock_region *reg;
14511463 	phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE);
14521464 	phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
1465
+	phys_addr_t start, end;
1466
+	u64 i;
14531467 
14541468 	/* Map all the lowmem memory banks. */
1455
-	for_each_memblock(memory, reg) {
1456
-		phys_addr_t start = reg->base;
1457
-		phys_addr_t end = start + reg->size;
1469
+	for_each_mem_range(i, &start, &end) {
14581470 		struct map_desc map;
1459
-
1460
-		if (memblock_is_nomap(reg))
1461
-			continue;
14621471 
14631472 		if (end > arm_lowmem_limit)
14641473 			end = arm_lowmem_limit;
....@@ -1510,8 +1519,7 @@
15101519 }
15111520 
15121521 #ifdef CONFIG_ARM_PV_FIXUP
1513
-extern unsigned long __atags_pointer;
1514
-typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata);
1522
+typedef void pgtables_remap(long long offset, unsigned long pgd);
15151523 pgtables_remap lpae_pgtables_remap_asm;
15161524 
15171525 /*
....@@ -1524,7 +1532,6 @@
15241532 	unsigned long pa_pgd;
15251533 	unsigned int cr, ttbcr;
15261534 	long long offset;
1527
-	void *boot_data;
15281535 
15291536 	if (!mdesc->pv_fixup)
15301537 		return;
....@@ -1541,7 +1548,6 @@
15411548 	 */
15421549 	lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm);
15431550 	pa_pgd = __pa(swapper_pg_dir);
1544
-	boot_data = __va(__atags_pointer);
15451551 	barrier();
15461552 
15471553 	pr_info("Switching physical address space to 0x%08llx\n",
....@@ -1577,7 +1583,7 @@
15771583 	 * needs to be assembly.  It's fairly simple, as we're using the
15781584 	 * temporary tables setup by the initial assembly code.
15791585 	 */
1580
-	lpae_pgtables_remap(offset, pa_pgd, boot_data);
1586
+	lpae_pgtables_remap(offset, pa_pgd);
15811587 
15821588 	/* Re-enable the caches and cacheable TLB walks */
15831589 	asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr));
....@@ -1660,9 +1666,6 @@
16601666 
16611667 	empty_zero_page = virt_to_page(zero_page);
16621668 	__flush_dcache_page(NULL, empty_zero_page);
1663
-
1664
-	/* Compute the virt/idmap offset, mostly for the sake of KVM */
1665
-	kimage_voffset = (unsigned long)&kimage_voffset - virt_to_idmap(&kimage_voffset);
16661669 }
16671670 
16681671 void __init early_mm_init(const struct machine_desc *mdesc)
....@@ -1670,3 +1673,17 @@
16701673 	build_mem_type_table();
16711674 	early_paging_init(mdesc);
16721675 }
1676
+
1677
+void set_pte_at(struct mm_struct *mm, unsigned long addr,
1678
+			      pte_t *ptep, pte_t pteval)
1679
+{
1680
+	unsigned long ext = 0;
1681
+
1682
+	if (addr < TASK_SIZE && pte_valid_user(pteval)) {
1683
+		if (!pte_special(pteval))
1684
+			__sync_icache_dcache(pteval);
1685
+		ext |= PTE_EXT_NG;
1686
+	}
1687
+
1688
+	set_pte_ext(ptep, pteval, ext);
1689
+}