forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c
....@@ -1,7 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -31,14 +31,13 @@
3131 #include <linux/fs.h>
3232 #include <linux/version.h>
3333 #include <linux/dma-mapping.h>
34
-#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE)
35
-#include <linux/dma-attrs.h>
36
-#endif /* LINUX_VERSION_CODE < 4.8.0 */
3734 #include <linux/dma-buf.h>
3835 #include <linux/shrinker.h>
3936 #include <linux/cache.h>
4037 #include <linux/memory_group_manager.h>
41
-
38
+#include <linux/math64.h>
39
+#include <linux/migrate.h>
40
+#include <linux/version.h>
4241 #include <mali_kbase.h>
4342 #include <mali_kbase_mem_linux.h>
4443 #include <tl/mali_kbase_tracepoints.h>
....@@ -86,23 +85,34 @@
8685 #define IR_THRESHOLD_STEPS (256u)
8786 
8887 #if MALI_USE_CSF
89
-static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx,
90
-			struct vm_area_struct *vma);
91
-static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx,
92
-			struct vm_area_struct *vma);
88
+static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma);
89
+static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma);
9390 #endif
9491 
95
-static int kbase_vmap_phy_pages(struct kbase_context *kctx,
96
-		struct kbase_va_region *reg, u64 offset_bytes, size_t size,
97
-		struct kbase_vmap_struct *map);
92
+static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_region *reg,
93
+				u64 offset_bytes, size_t size, struct kbase_vmap_struct *map,
94
+				kbase_vmap_flag vmap_flags);
9895 static void kbase_vunmap_phy_pages(struct kbase_context *kctx,
9996 		struct kbase_vmap_struct *map);
10097 
10198 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma);
10299 
103
-static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx,
104
-		struct kbase_va_region *reg,
105
-		u64 new_pages, u64 old_pages);
100
+static bool is_process_exiting(struct vm_area_struct *vma)
101
+{
102
+	/* PF_EXITING flag can't be reliably used here for the detection
103
+	 * of process exit, as 'mm_users' counter could still be non-zero
104
+	 * when all threads of the process have exited. Later when the
105
+	 * thread (which took a reference on the 'mm' of process that
106
+	 * exited) drops it reference, the vm_ops->close method would be
107
+	 * called for all the vmas (owned by 'mm' of process that exited)
108
+	 * but the PF_EXITING flag may not be neccessarily set for the
109
+	 * thread at that time.
110
+	 */
111
+	if (atomic_read(&vma->vm_mm->mm_users))
112
+		return false;
113
+
114
+	return true;
115
+}
106116 
107117 /* Retrieve the associated region pointer if the GPU address corresponds to
108118  * one of the event memory pages. The enclosing region, if found, shouldn't
....@@ -184,20 +194,12 @@
184194 			reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
185195 		return -EINVAL;
186196 
187
-	if (size > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES -
188
-			atomic_read(&kctx->permanent_mapped_pages))) {
189
-		dev_warn(kctx->kbdev->dev, "Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages",
190
-				(u64)size,
191
-				KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES,
192
-				atomic_read(&kctx->permanent_mapped_pages));
193
-		return -ENOMEM;
194
-	}
195
-
196197 	kern_mapping = kzalloc(sizeof(*kern_mapping), GFP_KERNEL);
197198 	if (!kern_mapping)
198199 		return -ENOMEM;
199200 
200
-	err = kbase_vmap_phy_pages(kctx, reg, 0u, size_bytes, kern_mapping);
201
+	err = kbase_vmap_phy_pages(kctx, reg, 0u, size_bytes, kern_mapping,
202
+				   KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING);
201203 	if (err < 0)
202204 		goto vmap_fail;
203205 
....@@ -205,7 +207,6 @@
205207 	reg->flags &= ~KBASE_REG_GROWABLE;
206208 
207209 	reg->cpu_alloc->permanent_map = kern_mapping;
208
-	atomic_add(size, &kctx->permanent_mapped_pages);
209210 
210211 	return 0;
211212 vmap_fail:
....@@ -221,13 +222,6 @@
221222 	kfree(alloc->permanent_map);
222223 
223224 	alloc->permanent_map = NULL;
224
-
225
-	/* Mappings are only done on cpu_alloc, so don't need to worry about
226
-	 * this being reduced a second time if a separate gpu_alloc is
227
-	 * freed
228
-	 */
229
-	WARN_ON(alloc->nents > atomic_read(&kctx->permanent_mapped_pages));
230
-	atomic_sub(alloc->nents, &kctx->permanent_mapped_pages);
231225 }
232226 
233227 void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx,
....@@ -291,9 +285,9 @@
291285 	 */
292286 }
293287 
294
-struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx,
295
-					u64 va_pages, u64 commit_pages,
296
-					u64 extension, u64 *flags, u64 *gpu_va)
288
+struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
289
+					u64 extension, u64 *flags, u64 *gpu_va,
290
+					enum kbase_caller_mmu_sync_info mmu_sync_info)
297291 {
298292 	int zone;
299293 	struct kbase_va_region *reg;
....@@ -310,19 +304,21 @@
310304 		va_pages, commit_pages, extension, *flags);
311305 
312306 #if MALI_USE_CSF
313
-	*gpu_va = 0; /* return 0 on failure */
307
+	if (!(*flags & BASE_MEM_FIXED))
308
+		*gpu_va = 0; /* return 0 on failure */
314309 #else
315310 	if (!(*flags & BASE_MEM_FLAG_MAP_FIXED))
316311 		*gpu_va = 0; /* return 0 on failure */
312
+#endif
317313 	else
318
-		dev_err(dev,
314
+		dev_dbg(dev,
319315 			"Keeping requested GPU VA of 0x%llx\n",
320316 			(unsigned long long)*gpu_va);
321
-#endif
322317 
323318 	if (!kbase_check_alloc_flags(*flags)) {
324319 		dev_warn(dev,
325
-				"kbase_mem_alloc called with bad flags (%llx)",
320
+				"%s called with bad flags (%llx)",
321
+				__func__,
326322 				(unsigned long long)*flags);
327323 		goto bad_flags;
328324 	}
....@@ -344,7 +340,8 @@
344340 	}
345341 	if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
346342 			!kbase_device_is_cpu_coherent(kctx->kbdev)) {
347
-		dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
343
+		dev_warn(dev, "%s call required coherent mem when unavailable",
344
+			__func__);
348345 		goto bad_flags;
349346 	}
350347 	if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
....@@ -367,7 +364,20 @@
367364 	if (*flags & BASE_MEM_SAME_VA) {
368365 		rbtree = &kctx->reg_rbtree_same;
369366 		zone = KBASE_REG_ZONE_SAME_VA;
370
-	} else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) {
367
+	}
368
+#if MALI_USE_CSF
369
+	/* fixed va_zone always exists */
370
+	else if (*flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE)) {
371
+		if (*flags & BASE_MEM_PROT_GPU_EX) {
372
+			rbtree = &kctx->reg_rbtree_exec_fixed;
373
+			zone = KBASE_REG_ZONE_EXEC_FIXED_VA;
374
+		} else {
375
+			rbtree = &kctx->reg_rbtree_fixed;
376
+			zone = KBASE_REG_ZONE_FIXED_VA;
377
+		}
378
+	}
379
+#endif
380
+	else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) {
371381 		rbtree = &kctx->reg_rbtree_exec;
372382 		zone = KBASE_REG_ZONE_EXEC_VA;
373383 	} else {
....@@ -375,8 +385,7 @@
375385 		zone = KBASE_REG_ZONE_CUSTOM_VA;
376386 	}
377387 
378
-	reg = kbase_alloc_free_region(rbtree, PFN_DOWN(*gpu_va),
379
-			va_pages, zone);
388
+	reg = kbase_alloc_free_region(kctx->kbdev, rbtree, PFN_DOWN(*gpu_va), va_pages, zone);
380389 
381390 	if (!reg) {
382391 		dev_err(dev, "Failed to allocate free region");
....@@ -387,7 +396,7 @@
387396 		goto invalid_flags;
388397 
389398 	if (kbase_reg_prepare_native(reg, kctx,
390
-				base_mem_group_id_get(*flags)) != 0) {
399
+				     kbase_mem_group_id_get(*flags)) != 0) {
391400 		dev_err(dev, "Failed to prepare region");
392401 		goto prepare_failed;
393402 	}
....@@ -469,7 +478,26 @@
469478 
470479 		*gpu_va = (u64) cookie;
471480 	} else /* we control the VA */ {
472
-		if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, 1) != 0) {
481
+		size_t align = 1;
482
+
483
+		if (kctx->kbdev->pagesize_2mb) {
484
+			/* If there's enough (> 33 bits) of GPU VA space, align to 2MB
485
+			* boundaries. The similar condition is used for mapping from
486
+			* the SAME_VA zone inside kbase_context_get_unmapped_area().
487
+			*/
488
+			if (kctx->kbdev->gpu_props.mmu.va_bits > 33) {
489
+				if (va_pages >= (SZ_2M / SZ_4K))
490
+					align = (SZ_2M / SZ_4K);
491
+			}
492
+			if (*gpu_va)
493
+				align = 1;
494
+#if !MALI_USE_CSF
495
+			if (reg->flags & KBASE_REG_TILER_ALIGN_TOP)
496
+				align = 1;
497
+#endif /* !MALI_USE_CSF */
498
+		}
499
+		if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align,
500
+				   mmu_sync_info) != 0) {
473501 			dev_warn(dev, "Failed to map memory on GPU");
474502 			kbase_gpu_vm_unlock(kctx);
475503 			goto no_mmap;
....@@ -490,6 +518,14 @@
490518 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
491519 
492520 	kbase_gpu_vm_unlock(kctx);
521
+
522
+#if MALI_USE_CSF
523
+	if (*flags & BASE_MEM_FIXABLE)
524
+		atomic64_inc(&kctx->num_fixable_allocs);
525
+	else if (*flags & BASE_MEM_FIXED)
526
+		atomic64_inc(&kctx->num_fixed_allocs);
527
+#endif
528
+
493529 	return reg;
494530 
495531 no_mmap:
....@@ -600,11 +636,18 @@
600636 #if MALI_USE_CSF
601637 		if (KBASE_REG_CSF_EVENT & reg->flags)
602638 			*out |= BASE_MEM_CSF_EVENT;
639
+		if (((KBASE_REG_ZONE_MASK & reg->flags) == KBASE_REG_ZONE_FIXED_VA) ||
640
+		    ((KBASE_REG_ZONE_MASK & reg->flags) == KBASE_REG_ZONE_EXEC_FIXED_VA)) {
641
+			if (KBASE_REG_FIXED_ADDRESS & reg->flags)
642
+				*out |= BASE_MEM_FIXED;
643
+			else
644
+				*out |= BASE_MEM_FIXABLE;
645
+		}
603646 #endif
604647 		if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags)
605648 			*out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE;
606649 
607
-		*out |= base_mem_group_id_set(reg->cpu_alloc->group_id);
650
+		*out |= kbase_mem_group_id_set(reg->cpu_alloc->group_id);
608651 
609652 		WARN(*out & ~BASE_MEM_FLAGS_QUERYABLE,
610653 				"BASE_MEM_FLAGS_QUERYABLE needs updating\n");
....@@ -629,24 +672,36 @@
629672  * @s:        Shrinker
630673  * @sc:       Shrinker control
631674  *
632
- * Return: Number of pages which can be freed.
675
+ * Return: Number of pages which can be freed or SHRINK_EMPTY if no page remains.
633676  */
634677 static
635678 unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s,
636679 		struct shrink_control *sc)
637680 {
638
-	struct kbase_context *kctx;
639
-
640
-	kctx = container_of(s, struct kbase_context, reclaim);
681
+	struct kbase_context *kctx = container_of(s, struct kbase_context, reclaim);
682
+	int evict_nents = atomic_read(&kctx->evict_nents);
683
+	unsigned long nr_freeable_items;
641684 
642685 	WARN((sc->gfp_mask & __GFP_ATOMIC),
643686 	     "Shrinkers cannot be called for GFP_ATOMIC allocations. Check kernel mm for problems. gfp_mask==%x\n",
644687 	     sc->gfp_mask);
645688 	WARN(in_atomic(),
646
-	     "Shrinker called whilst in atomic context. The caller must switch to using GFP_ATOMIC or similar. gfp_mask==%x\n",
689
+	     "Shrinker called in atomic context. The caller must use GFP_ATOMIC or similar, then Shrinkers must not be called. gfp_mask==%x\n",
647690 	     sc->gfp_mask);
648691 
649
-	return atomic_read(&kctx->evict_nents);
692
+	if (unlikely(evict_nents < 0)) {
693
+		dev_err(kctx->kbdev->dev, "invalid evict_nents(%d)", evict_nents);
694
+		nr_freeable_items = 0;
695
+	} else {
696
+		nr_freeable_items = evict_nents;
697
+	}
698
+
699
+#if KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE
700
+	if (nr_freeable_items == 0)
701
+		nr_freeable_items = SHRINK_EMPTY;
702
+#endif
703
+
704
+	return nr_freeable_items;
650705 }
651706 
652707 /**
....@@ -655,8 +710,8 @@
655710  * @s:        Shrinker
656711  * @sc:       Shrinker control
657712  *
658
- * Return: Number of pages freed (can be less then requested) or -1 if the
659
- * shrinker failed to free pages in its pool.
713
+ * Return: Number of pages freed (can be less then requested) or
714
+ *         SHRINK_STOP if reclaim isn't possible.
660715  *
661716  * Note:
662717  * This function accesses region structures without taking the region lock,
....@@ -684,17 +739,15 @@
684739 	list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
685740 		int err;
686741 
742
+		if (!alloc->reg)
743
+			continue;
744
+
687745 		err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
688746 				0, alloc->nents);
689
-		if (err != 0) {
690
-			/*
691
-			 * Failed to remove GPU mapping, tell the shrinker
692
-			 * to stop trying to shrink our slab even though we
693
-			 * have pages in it.
694
-			 */
695
-			freed = -1;
696
-			goto out_unlock;
697
-		}
747
+
748
+		/* Failed to remove GPU mapping, proceed to next one. */
749
+		if (err != 0)
750
+			continue;
698751 
699752 		/*
700753 		 * Update alloc->evicted before freeing the backing so the
....@@ -718,7 +771,7 @@
718771 		if (freed > sc->nr_to_scan)
719772 			break;
720773 	}
721
-out_unlock:
774
+
722775 	mutex_unlock(&kctx->jit_evict_lock);
723776 
724777 	return freed;
....@@ -738,7 +791,11 @@
738791 	 * struct shrinker does not define batch
739792 	 */
740793 	kctx->reclaim.batch = 0;
794
+#if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE
741795 	register_shrinker(&kctx->reclaim);
796
+#else
797
+	register_shrinker(&kctx->reclaim, "mali-mem");
798
+#endif
742799 	return 0;
743800 }
744801 
....@@ -802,6 +859,9 @@
802859 
803860 	lockdep_assert_held(&kctx->reg_lock);
804861 
862
+	/* Memory is in the process of transitioning to the shrinker, and
863
+	 * should ignore migration attempts
864
+	 */
805865 	kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg,
806866 			0, gpu_alloc->nents);
807867 
....@@ -809,12 +869,17 @@
809869 	/* This allocation can't already be on a list. */
810870 	WARN_ON(!list_empty(&gpu_alloc->evict_node));
811871 
812
-	/*
813
-	 * Add the allocation to the eviction list, after this point the shrink
872
+	/* Add the allocation to the eviction list, after this point the shrink
814873 	 * can reclaim it.
815874 	 */
816875 	list_add(&gpu_alloc->evict_node, &kctx->evict_list);
817876 	atomic_add(gpu_alloc->nents, &kctx->evict_nents);
877
+
878
+	/* Indicate to page migration that the memory can be reclaimed by the shrinker.
879
+	 */
880
+	if (kbase_page_migration_enabled)
881
+		kbase_set_phy_alloc_page_status(gpu_alloc, NOT_MOVABLE);
882
+
818883 	mutex_unlock(&kctx->jit_evict_lock);
819884 	kbase_mem_evictable_mark_reclaim(gpu_alloc);
820885 
....@@ -826,6 +891,11 @@
826891 {
827892 	struct kbase_context *kctx = gpu_alloc->imported.native.kctx;
828893 	int err = 0;
894
+
895
+	/* Calls to this function are inherently asynchronous, with respect to
896
+	 * MMU operations.
897
+	 */
898
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
829899 
830900 	lockdep_assert_held(&kctx->reg_lock);
831901 
....@@ -856,11 +926,20 @@
856926 			 * pre-eviction size.
857927 			 */
858928 			if (!err)
859
-				err = kbase_mem_grow_gpu_mapping(kctx,
860
-						gpu_alloc->reg,
861
-						gpu_alloc->evicted, 0);
929
+				err = kbase_mem_grow_gpu_mapping(
930
+					kctx, gpu_alloc->reg,
931
+					gpu_alloc->evicted, 0, mmu_sync_info);
862932 
863933 			gpu_alloc->evicted = 0;
934
+
935
+			/* Since the allocation is no longer evictable, and we ensure that
936
+			 * it grows back to its pre-eviction size, we will consider the
937
+			 * state of it to be ALLOCATED_MAPPED, as that is the only state
938
+			 * in which a physical allocation could transition to NOT_MOVABLE
939
+			 * from.
940
+			 */
941
+			if (kbase_page_migration_enabled)
942
+				kbase_set_phy_alloc_page_status(gpu_alloc, ALLOCATED_MAPPED);
864943 		}
865944 	}
866945 
....@@ -911,6 +990,15 @@
911990 	/* Validate the region */
912991 	reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr);
913992 	if (kbase_is_region_invalid_or_free(reg))
993
+		goto out_unlock;
994
+
995
+	/* There is no use case to support MEM_FLAGS_CHANGE ioctl for allocations
996
+	 * that have NO_USER_FREE flag set, to mark them as evictable/reclaimable.
997
+	 * This would usually include JIT allocations, Tiler heap related allocations
998
+	 * & GPU queue ringbuffer and none of them needs to be explicitly marked
999
+	 * as evictable by Userspace.
1000
+	 */
1001
+	if (kbase_va_region_is_no_user_free(reg))
9141002 		goto out_unlock;
9151003 
9161004 	/* Is the region being transitioning between not needed and needed? */
....@@ -1022,7 +1110,7 @@
10221110 		struct kbase_va_region *reg, enum kbase_sync_type sync_fn)
10231111 {
10241112 	int ret = -EINVAL;
1025
-	struct dma_buf *dma_buf;
1113
+	struct dma_buf __maybe_unused *dma_buf;
10261114 	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
10271115 
10281116 	lockdep_assert_held(&kctx->reg_lock);
....@@ -1066,19 +1154,7 @@
10661154 			ret = 0;
10671155 		}
10681156 #else
1069
-	/* Though the below version check could be superfluous depending upon the version condition
1070
-	 * used for enabling KBASE_MEM_ION_SYNC_WORKAROUND, we still keep this check here to allow
1071
-	 * ease of modification for non-ION systems or systems where ION has been patched.
1072
-	 */
1073
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS)
1074
-		dma_buf_end_cpu_access(dma_buf,
1075
-				0, dma_buf->size,
1076
-				dir);
1077
-		ret = 0;
1078
-#else
1079
-		ret = dma_buf_end_cpu_access(dma_buf,
1080
-				dir);
1081
-#endif
1157
+		ret = dma_buf_end_cpu_access(dma_buf, dir);
10821158 #endif /* KBASE_MEM_ION_SYNC_WORKAROUND */
10831159 		break;
10841160 	case KBASE_SYNC_TO_CPU:
....@@ -1095,11 +1171,7 @@
10951171 			ret = 0;
10961172 		}
10971173 #else
1098
-		ret = dma_buf_begin_cpu_access(dma_buf,
1099
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS)
1100
-				0, dma_buf->size,
1101
-#endif
1102
-				dir);
1174
+		ret = dma_buf_begin_cpu_access(dma_buf, dir);
11031175 #endif /* KBASE_MEM_ION_SYNC_WORKAROUND */
11041176 		break;
11051177 	}
....@@ -1218,6 +1290,11 @@
12181290 	struct kbase_mem_phy_alloc *alloc;
12191291 	unsigned long gwt_mask = ~0;
12201292 
1293
+	/* Calls to this function are inherently asynchronous, with respect to
1294
+	 * MMU operations.
1295
+	 */
1296
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
1297
+
12211298 	lockdep_assert_held(&kctx->reg_lock);
12221299 
12231300 	alloc = reg->gpu_alloc;
....@@ -1244,14 +1321,11 @@
12441321 		gwt_mask = ~KBASE_REG_GPU_WR;
12451322 #endif
12461323 
1247
-	err = kbase_mmu_insert_pages(kctx->kbdev,
1248
-				     &kctx->mmu,
1249
-				     reg->start_pfn,
1250
-				     kbase_get_gpu_phy_pages(reg),
1251
-				     kbase_reg_current_backed_size(reg),
1252
-				     reg->flags & gwt_mask,
1253
-				     kctx->as_nr,
1254
-				     alloc->group_id);
1324
+	err = kbase_mmu_insert_imported_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn,
1325
+					      kbase_get_gpu_phy_pages(reg),
1326
+					      kbase_reg_current_backed_size(reg),
1327
+					      reg->flags & gwt_mask, kctx->as_nr, alloc->group_id,
1328
+					      mmu_sync_info, NULL);
12551329 	if (err)
12561330 		goto bad_insert;
12571331 
....@@ -1264,13 +1338,11 @@
12641338 		 * Assume alloc->nents is the number of actual pages in the
12651339 		 * dma-buf memory.
12661340 		 */
1267
-		err = kbase_mmu_insert_single_page(kctx,
1268
-				reg->start_pfn + alloc->nents,
1269
-				kctx->aliasing_sink_page,
1270
-				reg->nr_pages - alloc->nents,
1271
-				(reg->flags | KBASE_REG_GPU_RD) &
1272
-				~KBASE_REG_GPU_WR,
1273
-				KBASE_MEM_GROUP_SINK);
1341
+		err = kbase_mmu_insert_single_imported_page(
1342
+			kctx, reg->start_pfn + alloc->nents, kctx->aliasing_sink_page,
1343
+			reg->nr_pages - alloc->nents,
1344
+			(reg->flags | KBASE_REG_GPU_RD) & ~KBASE_REG_GPU_WR, KBASE_MEM_GROUP_SINK,
1345
+			mmu_sync_info);
12741346 		if (err)
12751347 			goto bad_pad_insert;
12761348 	}
....@@ -1278,11 +1350,8 @@
12781350 	return 0;
12791351 
12801352 bad_pad_insert:
1281
-	kbase_mmu_teardown_pages(kctx->kbdev,
1282
-				 &kctx->mmu,
1283
-				 reg->start_pfn,
1284
-				 alloc->nents,
1285
-				 kctx->as_nr);
1353
+	kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn, alloc->pages,
1354
+				 alloc->nents, alloc->nents, kctx->as_nr, true);
12861355 bad_insert:
12871356 	kbase_mem_umm_unmap_attachment(kctx, alloc);
12881357 bad_map_attachment:
....@@ -1310,11 +1379,9 @@
13101379 	if (!kbase_is_region_invalid_or_free(reg) && reg->gpu_alloc == alloc) {
13111380 		int err;
13121381 
1313
-		err = kbase_mmu_teardown_pages(kctx->kbdev,
1314
-					       &kctx->mmu,
1315
-					       reg->start_pfn,
1316
-					       reg->nr_pages,
1317
-					       kctx->as_nr);
1382
+		err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn,
1383
+					       alloc->pages, reg->nr_pages, reg->nr_pages,
1384
+					       kctx->as_nr, true);
13181385 		WARN_ON(err);
13191386 	}
13201387 
....@@ -1386,6 +1453,9 @@
13861453 		return NULL;
13871454 	}
13881455 
1456
+	if (!kbase_import_size_is_valid(kctx->kbdev, *va_pages))
1457
+		return NULL;
1458
+
13891459 	/* ignore SAME_VA */
13901460 	*flags &= ~BASE_MEM_SAME_VA;
13911461 
....@@ -1406,23 +1476,21 @@
14061476 	if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP)
14071477 		need_sync = true;
14081478 
1409
-#if IS_ENABLED(CONFIG_64BIT)
1410
-	if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1479
+	if (!kbase_ctx_compat_mode(kctx)) {
14111480 		/*
14121481 		 * 64-bit tasks require us to reserve VA on the CPU that we use
14131482 		 * on the GPU.
14141483 		 */
14151484 		shared_zone = true;
14161485 	}
1417
-#endif
14181486 
14191487 	if (shared_zone) {
14201488 		*flags |= BASE_MEM_NEED_MMAP;
1421
-		reg = kbase_alloc_free_region(&kctx->reg_rbtree_same,
1422
-				0, *va_pages, KBASE_REG_ZONE_SAME_VA);
1489
+		reg = kbase_alloc_free_region(kctx->kbdev, &kctx->reg_rbtree_same, 0, *va_pages,
1490
+					      KBASE_REG_ZONE_SAME_VA);
14231491 	} else {
1424
-		reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom,
1425
-				0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA);
1492
+		reg = kbase_alloc_free_region(kctx->kbdev, &kctx->reg_rbtree_custom, 0, *va_pages,
1493
+					      KBASE_REG_ZONE_CUSTOM_VA);
14261494 	}
14271495 
14281496 	if (!reg) {
....@@ -1507,7 +1575,7 @@
15071575 		struct kbase_context *kctx, unsigned long address,
15081576 		unsigned long size, u64 *va_pages, u64 *flags)
15091577 {
1510
-	long i;
1578
+	long i, dma_mapped_pages;
15111579 	struct kbase_va_region *reg;
15121580 	struct rb_root *rbtree;
15131581 	long faulted_pages;
....@@ -1516,6 +1584,8 @@
15161584 	u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx->kbdev);
15171585 	struct kbase_alloc_import_user_buf *user_buf;
15181586 	struct page **pages = NULL;
1587
+	struct tagged_addr *pa;
1588
+	struct device *dev;
15191589 	int write;
15201590 
15211591 	/* Flag supported only for dma-buf imported memory */
....@@ -1553,21 +1623,22 @@
15531623 		/* 64-bit address range is the max */
15541624 		goto bad_size;
15551625 
1626
+	if (!kbase_import_size_is_valid(kctx->kbdev, *va_pages))
1627
+		goto bad_size;
1628
+
15561629 	/* SAME_VA generally not supported with imported memory (no known use cases) */
15571630 	*flags &= ~BASE_MEM_SAME_VA;
15581631 
15591632 	if (*flags & BASE_MEM_IMPORT_SHARED)
15601633 		shared_zone = true;
15611634 
1562
-#if IS_ENABLED(CONFIG_64BIT)
1563
-	if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1635
+	if (!kbase_ctx_compat_mode(kctx)) {
15641636 		/*
15651637 		 * 64-bit tasks require us to reserve VA on the CPU that we use
15661638 		 * on the GPU.
15671639 		 */
15681640 		shared_zone = true;
15691641 	}
1570
-#endif
15711642 
15721643 	if (shared_zone) {
15731644 		*flags |= BASE_MEM_NEED_MMAP;
....@@ -1576,7 +1647,7 @@
15761647 	} else
15771648 		rbtree = &kctx->reg_rbtree_custom;
15781649 
1579
-	reg = kbase_alloc_free_region(rbtree, 0, *va_pages, zone);
1650
+	reg = kbase_alloc_free_region(kctx->kbdev, rbtree, 0, *va_pages, zone);
15801651 
15811652 	if (!reg)
15821653 		goto no_region;
....@@ -1602,11 +1673,7 @@
16021673 	user_buf->address = address;
16031674 	user_buf->nr_pages = *va_pages;
16041675 	user_buf->mm = current->mm;
1605
-#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1606
-	atomic_inc(&current->mm->mm_count);
1607
-#else
1608
-	mmgrab(current->mm);
1609
-#endif
1676
+	kbase_mem_mmgrab();
16101677 	if (reg->gpu_alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE)
16111678 		user_buf->pages = vmalloc(*va_pages * sizeof(struct page *));
16121679 	else
....@@ -1632,20 +1699,21 @@
16321699 
16331700 	write = reg->flags & (KBASE_REG_CPU_WR | KBASE_REG_GPU_WR);
16341701 
1635
-#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE
1636
-	faulted_pages = get_user_pages(current, current->mm, address, *va_pages,
1637
-#if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \
1638
-KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE
1639
-			write ? FOLL_WRITE : 0, pages, NULL);
1640
-#else
1641
-			write, 0, pages, NULL);
1642
-#endif
1643
-#elif KERNEL_VERSION(4, 9, 0) > LINUX_VERSION_CODE
1644
-	faulted_pages = get_user_pages(address, *va_pages,
1645
-			write, 0, pages, NULL);
1646
-#else
1702
+#if KERNEL_VERSION(5, 9, 0) > LINUX_VERSION_CODE
16471703 	faulted_pages = get_user_pages(address, *va_pages,
16481704 			write ? FOLL_WRITE : 0, pages, NULL);
1705
+#else
1706
+	/* pin_user_pages function cannot be called with pages param NULL.
1707
+	 * get_user_pages function will be used instead because it is safe to be
1708
+	 * used with NULL pages param as long as it doesn't have FOLL_GET flag.
1709
+	 */
1710
+	if (pages != NULL) {
1711
+		faulted_pages =
1712
+			pin_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, pages, NULL);
1713
+	} else {
1714
+		faulted_pages =
1715
+			get_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, pages, NULL);
1716
+	}
16491717 #endif
16501718 
16511719 	up_read(kbase_mem_get_process_mmap_lock());
....@@ -1656,31 +1724,44 @@
16561724 	reg->gpu_alloc->nents = 0;
16571725 	reg->extension = 0;
16581726 
1659
-	if (pages) {
1660
-		struct device *dev = kctx->kbdev->dev;
1661
-		unsigned long local_size = user_buf->size;
1662
-		unsigned long offset = user_buf->address & ~PAGE_MASK;
1663
-		struct tagged_addr *pa = kbase_get_gpu_phy_pages(reg);
1727
+	pa = kbase_get_gpu_phy_pages(reg);
1728
+	dev = kctx->kbdev->dev;
16641729 
1730
+	if (pages) {
16651731 		/* Top bit signifies that this was pinned on import */
16661732 		user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT;
16671733 
1734
+		/* Manual CPU cache synchronization.
1735
+		 *
1736
+		 * The driver disables automatic CPU cache synchronization because the
1737
+		 * memory pages that enclose the imported region may also contain
1738
+		 * sub-regions which are not imported and that are allocated and used
1739
+		 * by the user process. This may be the case of memory at the beginning
1740
+		 * of the first page and at the end of the last page. Automatic CPU cache
1741
+		 * synchronization would force some operations on those memory allocations,
1742
+		 * unbeknown to the user process: in particular, a CPU cache invalidate
1743
+		 * upon unmapping would destroy the content of dirty CPU caches and cause
1744
+		 * the user process to lose CPU writes to the non-imported sub-regions.
1745
+		 *
1746
+		 * When the GPU claims ownership of the imported memory buffer, it shall
1747
+		 * commit CPU writes for the whole of all pages that enclose the imported
1748
+		 * region, otherwise the initial content of memory would be wrong.
1749
+		 */
16681750 		for (i = 0; i < faulted_pages; i++) {
16691751 			dma_addr_t dma_addr;
1670
-			unsigned long min;
1671
-
1672
-			min = MIN(PAGE_SIZE - offset, local_size);
1673
-			dma_addr = dma_map_page(dev, pages[i],
1674
-					offset, min,
1675
-					DMA_BIDIRECTIONAL);
1752
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
1753
+			dma_addr = dma_map_page(dev, pages[i], 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
1754
+#else
1755
+			dma_addr = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE,
1756
+						      DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
1757
+#endif
16761758 			if (dma_mapping_error(dev, dma_addr))
16771759 				goto unwind_dma_map;
16781760 
16791761 			user_buf->dma_addrs[i] = dma_addr;
16801762 			pa[i] = as_tagged(page_to_phys(pages[i]));
16811763 
1682
-			local_size -= min;
1683
-			offset = 0;
1764
+			dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
16841765 		}
16851766 
16861767 		reg->gpu_alloc->nents = faulted_pages;
....@@ -1689,15 +1770,32 @@
16891770 	return reg;
16901771 
16911772 unwind_dma_map:
1692
-	while (i--) {
1693
-		dma_unmap_page(kctx->kbdev->dev,
1694
-				user_buf->dma_addrs[i],
1695
-				PAGE_SIZE, DMA_BIDIRECTIONAL);
1773
+	dma_mapped_pages = i;
1774
+	/* Run the unmap loop in the same order as map loop, and perform again
1775
+	 * CPU cache synchronization to re-write the content of dirty CPU caches
1776
+	 * to memory. This precautionary measure is kept here to keep this code
1777
+	 * aligned with kbase_jd_user_buf_map() to allow for a potential refactor
1778
+	 * in the future.
1779
+	 */
1780
+	for (i = 0; i < dma_mapped_pages; i++) {
1781
+		dma_addr_t dma_addr = user_buf->dma_addrs[i];
1782
+
1783
+		dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
1784
+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
1785
+		dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
1786
+#else
1787
+		dma_unmap_page_attrs(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL,
1788
+				     DMA_ATTR_SKIP_CPU_SYNC);
1789
+#endif
16961790 	}
16971791 fault_mismatch:
16981792 	if (pages) {
1793
+		/* In this case, the region was not yet in the region tracker,
1794
+		 * and so there are no CPU mappings to remove before we unpin
1795
+		 * the page
1796
+		 */
16991797 		for (i = 0; i < faulted_pages; i++)
1700
-			put_page(pages[i]);
1798
+			kbase_unpin_user_buf_page(pages[i]);
17011799 	}
17021800 no_page_array:
17031801 invalid_flags:
....@@ -1708,7 +1806,6 @@
17081806 no_region:
17091807 bad_size:
17101808 	return NULL;
1711
-
17121809 }
17131810 
17141811 
....@@ -1720,6 +1817,12 @@
17201817 	u64 gpu_va;
17211818 	size_t i;
17221819 	bool coherent;
1820
+	uint64_t max_stride;
1821
+
1822
+	/* Calls to this function are inherently asynchronous, with respect to
1823
+	 * MMU operations.
1824
+	 */
1825
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
17231826 
17241827 	KBASE_DEBUG_ASSERT(kctx);
17251828 	KBASE_DEBUG_ASSERT(flags);
....@@ -1733,7 +1836,8 @@
17331836 
17341837 	if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
17351838 		dev_warn(kctx->kbdev->dev,
1736
-				"kbase_mem_alias called with bad flags (%llx)",
1839
+				"%s called with bad flags (%llx)",
1840
+				__func__,
17371841 				(unsigned long long)*flags);
17381842 		goto bad_flags;
17391843 	}
....@@ -1746,6 +1850,11 @@
17461850 	if (!nents)
17471851 		goto bad_nents;
17481852 
1853
+	max_stride = div64_u64(U64_MAX, nents);
1854
+
1855
+	if (stride > max_stride)
1856
+		goto bad_size;
1857
+
17491858 	if ((nents * stride) > (U64_MAX / PAGE_SIZE))
17501859 		/* 64-bit address range is the max */
17511860 		goto bad_size;
....@@ -1753,22 +1862,19 @@
17531862 	/* calculate the number of pages this alias will cover */
17541863 	*num_pages = nents * stride;
17551864 
1756
-#if IS_ENABLED(CONFIG_64BIT)
1757
-	if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1865
+	if (!kbase_alias_size_is_valid(kctx->kbdev, *num_pages))
1866
+		goto bad_size;
1867
+
1868
+	if (!kbase_ctx_compat_mode(kctx)) {
17581869 		/* 64-bit tasks must MMAP anyway, but not expose this address to
17591870 		 * clients
17601871 		 */
17611872 		*flags |= BASE_MEM_NEED_MMAP;
1762
-		reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0,
1763
-				*num_pages,
1764
-				KBASE_REG_ZONE_SAME_VA);
1873
+		reg = kbase_alloc_free_region(kctx->kbdev, &kctx->reg_rbtree_same, 0, *num_pages,
1874
+					      KBASE_REG_ZONE_SAME_VA);
17651875 	} else {
1766
-#else
1767
-	if (1) {
1768
-#endif
1769
-		reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom,
1770
-				0, *num_pages,
1771
-				KBASE_REG_ZONE_CUSTOM_VA);
1876
+		reg = kbase_alloc_free_region(kctx->kbdev, &kctx->reg_rbtree_custom, 0, *num_pages,
1877
+					      KBASE_REG_ZONE_CUSTOM_VA);
17721878 	}
17731879 
17741880 	if (!reg)
....@@ -1817,9 +1923,9 @@
18171923 			/* validate found region */
18181924 			if (kbase_is_region_invalid_or_free(aliasing_reg))
18191925 				goto bad_handle; /* Not found/already free */
1820
-			if (aliasing_reg->flags & KBASE_REG_DONT_NEED)
1926
+			if (kbase_is_region_shrinkable(aliasing_reg))
18211927 				goto bad_handle; /* Ephemeral region */
1822
-			if (aliasing_reg->flags & KBASE_REG_NO_USER_FREE)
1928
+			if (kbase_va_region_is_no_user_free(aliasing_reg))
18231929 				goto bad_handle; /* JIT regions can't be
18241930 						  * aliased. NO_USER_FREE flag
18251931 						  * covers the entire lifetime
....@@ -1874,8 +1980,7 @@
18741980 		}
18751981 	}
18761982 
1877
-#if IS_ENABLED(CONFIG_64BIT)
1878
-	if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) {
1983
+	if (!kbase_ctx_compat_mode(kctx)) {
18791984 		/* Bind to a cookie */
18801985 		if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) {
18811986 			dev_err(kctx->kbdev->dev, "No cookies available for allocation!");
....@@ -1890,11 +1995,10 @@
18901995 		/* relocate to correct base */
18911996 		gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE);
18921997 		gpu_va <<= PAGE_SHIFT;
1893
-	} else /* we control the VA */ {
1894
-#else
1895
-	if (1) {
1896
-#endif
1897
-		if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) {
1998
+	} else {
1999
+		/* we control the VA */
2000
+		if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1,
2001
+				   mmu_sync_info) != 0) {
18982002 			dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU");
18992003 			goto no_mmap;
19002004 		}
....@@ -1909,9 +2013,7 @@
19092013 
19102014 	return gpu_va;
19112015 
1912
-#if IS_ENABLED(CONFIG_64BIT)
19132016 no_cookie:
1914
-#endif
19152017 no_mmap:
19162018 bad_handle:
19172019 	/* Marking the source allocs as not being mapped on the GPU and putting
....@@ -1939,6 +2041,11 @@
19392041 {
19402042 	struct kbase_va_region *reg;
19412043 
2044
+	/* Calls to this function are inherently asynchronous, with respect to
2045
+	 * MMU operations.
2046
+	 */
2047
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
2048
+
19422049 	KBASE_DEBUG_ASSERT(kctx);
19432050 	KBASE_DEBUG_ASSERT(gpu_va);
19442051 	KBASE_DEBUG_ASSERT(va_pages);
....@@ -1950,7 +2057,8 @@
19502057 
19512058 	if (!kbase_check_import_flags(*flags)) {
19522059 		dev_warn(kctx->kbdev->dev,
1953
-				"kbase_mem_import called with bad flags (%llx)",
2060
+				"%s called with bad flags (%llx)",
2061
+				__func__,
19542062 				(unsigned long long)*flags);
19552063 		goto bad_flags;
19562064 	}
....@@ -1963,7 +2071,8 @@
19632071 	if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
19642072 			!kbase_device_is_cpu_coherent(kctx->kbdev)) {
19652073 		dev_warn(kctx->kbdev->dev,
1966
-				"kbase_mem_import call required coherent mem when unavailable");
2074
+				"%s call required coherent mem when unavailable",
2075
+				__func__);
19672076 		goto bad_flags;
19682077 	}
19692078 	if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
....@@ -1971,7 +2080,10 @@
19712080 		/* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */
19722081 		*flags &= ~BASE_MEM_COHERENT_SYSTEM;
19732082 	}
1974
-
2083
+	if (((*flags & BASE_MEM_CACHED_CPU) == 0) && (type == BASE_MEM_IMPORT_TYPE_USER_BUFFER)) {
2084
+		dev_warn(kctx->kbdev->dev, "USER_BUFFER must be CPU cached");
2085
+		goto bad_flags;
2086
+	}
19752087 	if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) {
19762088 		dev_warn(kctx->kbdev->dev,
19772089 				"padding is only supported for UMM");
....@@ -2038,7 +2150,8 @@
20382150 
20392151 	} else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES)  {
20402152 		/* we control the VA, mmap now to the GPU */
2041
-		if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0)
2153
+		if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1, mmu_sync_info) !=
2154
+		    0)
20422155 			goto no_gpu_va;
20432156 		/* return real GPU VA */
20442157 		*gpu_va = reg->start_pfn << PAGE_SHIFT;
....@@ -2072,8 +2185,9 @@
20722185 }
20732186 
20742187 int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx,
2075
-		struct kbase_va_region *reg,
2076
-		u64 new_pages, u64 old_pages)
2188
+			       struct kbase_va_region *reg, u64 new_pages,
2189
+			       u64 old_pages,
2190
+			       enum kbase_caller_mmu_sync_info mmu_sync_info)
20772191 {
20782192 	struct tagged_addr *phy_pages;
20792193 	u64 delta = new_pages - old_pages;
....@@ -2083,9 +2197,9 @@
20832197 
20842198 	/* Map the new pages into the GPU */
20852199 	phy_pages = kbase_get_gpu_phy_pages(reg);
2086
-	ret = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu,
2087
-		reg->start_pfn + old_pages, phy_pages + old_pages, delta,
2088
-		reg->flags, kctx->as_nr, reg->gpu_alloc->group_id);
2200
+	ret = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn + old_pages,
2201
+				     phy_pages + old_pages, delta, reg->flags, kctx->as_nr,
2202
+				     reg->gpu_alloc->group_id, mmu_sync_info, reg, false);
20892203 
20902204 	return ret;
20912205 }
....@@ -2105,28 +2219,16 @@
21052219 			(old_pages - new_pages)<<PAGE_SHIFT, 1);
21062220 }
21072221 
2108
-/**
2109
- * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation
2110
- * @kctx:      Context the region belongs to
2111
- * @reg:       The GPU region or NULL if there isn't one
2112
- * @new_pages: The number of pages after the shrink
2113
- * @old_pages: The number of pages before the shrink
2114
- *
2115
- * Return: 0 on success, negative -errno on error
2116
- *
2117
- * Unmap the shrunk pages from the GPU mapping. Note that the size of the region
2118
- * itself is unmodified as we still need to reserve the VA, only the page tables
2119
- * will be modified by this function.
2120
- */
2121
-static int kbase_mem_shrink_gpu_mapping(struct kbase_context *const kctx,
2122
-		struct kbase_va_region *const reg,
2123
-		u64 const new_pages, u64 const old_pages)
2222
+int kbase_mem_shrink_gpu_mapping(struct kbase_context *const kctx,
2223
+				 struct kbase_va_region *const reg, u64 const new_pages,
2224
+				 u64 const old_pages)
21242225 {
21252226 	u64 delta = old_pages - new_pages;
2227
+	struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc;
21262228 	int ret = 0;
21272229 
2128
-	ret = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu,
2129
-			reg->start_pfn + new_pages, delta, kctx->as_nr);
2230
+	ret = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn + new_pages,
2231
+				       alloc->pages + new_pages, delta, delta, kctx->as_nr, false);
21302232 
21312233 	return ret;
21322234 }
....@@ -2138,6 +2240,11 @@
21382240 	int res = -EINVAL;
21392241 	struct kbase_va_region *reg;
21402242 	bool read_locked = false;
2243
+
2244
+	/* Calls to this function are inherently asynchronous, with respect to
2245
+	 * MMU operations.
2246
+	 */
2247
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
21412248 
21422249 	KBASE_DEBUG_ASSERT(kctx);
21432250 	KBASE_DEBUG_ASSERT(gpu_addr != 0);
....@@ -2185,8 +2292,11 @@
21852292 
21862293 	if (atomic_read(&reg->cpu_alloc->kernel_mappings) > 0)
21872294 		goto out_unlock;
2188
-	/* can't grow regions which are ephemeral */
2189
-	if (reg->flags & KBASE_REG_DONT_NEED)
2295
+
2296
+	if (kbase_is_region_shrinkable(reg))
2297
+		goto out_unlock;
2298
+
2299
+	if (kbase_va_region_is_no_user_free(reg))
21902300 		goto out_unlock;
21912301 
21922302 #ifdef CONFIG_MALI_MEMORY_FULLY_BACKED
....@@ -2230,8 +2340,8 @@
22302340 		/* No update required for CPU mappings, that's done on fault. */
22312341 
22322342 		/* Update GPU mapping. */
2233
-		res = kbase_mem_grow_gpu_mapping(kctx, reg,
2234
-				new_pages, old_pages);
2343
+		res = kbase_mem_grow_gpu_mapping(kctx, reg, new_pages,
2344
+						 old_pages, mmu_sync_info);
22352345 
22362346 		/* On error free the new pages */
22372347 		if (res) {
....@@ -2259,7 +2369,7 @@
22592369 }
22602370 
22612371 int kbase_mem_shrink(struct kbase_context *const kctx,
2262
-		struct kbase_va_region *const reg, u64 const new_pages)
2372
+		struct kbase_va_region *const reg, u64 new_pages)
22632373 {
22642374 	u64 delta, old_pages;
22652375 	int err;
....@@ -2289,6 +2399,19 @@
22892399 		kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
22902400 		if (reg->cpu_alloc != reg->gpu_alloc)
22912401 			kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
2402
+
2403
+		if (kctx->kbdev->pagesize_2mb) {
2404
+			if (kbase_reg_current_backed_size(reg) > new_pages) {
2405
+				old_pages = new_pages;
2406
+				new_pages = kbase_reg_current_backed_size(reg);
2407
+
2408
+				/* Update GPU mapping. */
2409
+				err = kbase_mem_grow_gpu_mapping(kctx, reg, new_pages, old_pages,
2410
+								 CALLER_MMU_ASYNC);
2411
+			}
2412
+		} else {
2413
+			WARN_ON(kbase_reg_current_backed_size(reg) != new_pages);
2414
+		}
22922415 	}
22932416 
22942417 	return err;
....@@ -2327,7 +2450,7 @@
23272450 		/* Avoid freeing memory on the process death which results in
23282451 		 * GPU Page Fault. Memory will be freed in kbase_destroy_context
23292452 		 */
2330
-		if (!(current->flags & PF_EXITING))
2453
+		if (!is_process_exiting(vma))
23312454 			kbase_mem_free_region(map->kctx, map->region);
23322455 	}
23332456 
....@@ -2559,7 +2682,6 @@
25592682 	while (kbase_jit_evict(kctx))
25602683 		;
25612684 }
2562
-#endif
25632685 
25642686 static int kbase_mmu_dump_mmap(struct kbase_context *kctx,
25652687 			struct vm_area_struct *vma,
....@@ -2572,13 +2694,13 @@
25722694 	size_t size;
25732695 	int err = 0;
25742696 
2575
-	dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
2697
+	lockdep_assert_held(&kctx->reg_lock);
2698
+
2699
+	dev_dbg(kctx->kbdev->dev, "%s\n", __func__);
25762700 	size = (vma->vm_end - vma->vm_start);
25772701 	nr_pages = size >> PAGE_SHIFT;
25782702 
2579
-#ifdef CONFIG_MALI_VECTOR_DUMP
25802703 	kbase_free_unused_jit_allocations(kctx);
2581
-#endif
25822704 
25832705 	kaddr = kbase_mmu_dump(kctx, nr_pages);
25842706 
....@@ -2587,8 +2709,8 @@
25872709 		goto out;
25882710 	}
25892711 
2590
-	new_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, nr_pages,
2591
-			KBASE_REG_ZONE_SAME_VA);
2712
+	new_reg = kbase_alloc_free_region(kctx->kbdev, &kctx->reg_rbtree_same, 0, nr_pages,
2713
+					  KBASE_REG_ZONE_SAME_VA);
25922714 	if (!new_reg) {
25932715 		err = -ENOMEM;
25942716 		WARN_ON(1);
....@@ -2617,7 +2739,7 @@
26172739 	*kmap_addr = kaddr;
26182740 	*reg = new_reg;
26192741 
2620
-	dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
2742
+	dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
26212743 	return 0;
26222744 
26232745 out_no_alloc:
....@@ -2626,7 +2748,7 @@
26262748 out:
26272749 	return err;
26282750 }
2629
-
2751
+#endif
26302752 
26312753 void kbase_os_mem_map_lock(struct kbase_context *kctx)
26322754 {
....@@ -2646,13 +2768,18 @@
26462768 			   size_t *nr_pages, size_t *aligned_offset)
26472769 
26482770 {
2649
-	int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
2771
+	unsigned int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE);
26502772 	struct kbase_va_region *reg;
26512773 	int err = 0;
26522774 
2775
+	/* Calls to this function are inherently asynchronous, with respect to
2776
+	 * MMU operations.
2777
+	 */
2778
+	const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC;
2779
+
26532780 	*aligned_offset = 0;
26542781 
2655
-	dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n");
2782
+	dev_dbg(kctx->kbdev->dev, "%s\n", __func__);
26562783 
26572784 	/* SAME_VA stuff, fetch the right region */
26582785 	reg = kctx->pending_regions[cookie];
....@@ -2682,9 +2809,8 @@
26822809 
26832810 	/* adjust down nr_pages to what we have physically */
26842811 	*nr_pages = kbase_reg_current_backed_size(reg);
2685
-
26862812 	if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset,
2687
-						reg->nr_pages, 1) != 0) {
2813
+			   reg->nr_pages, 1, mmu_sync_info) != 0) {
26882814 		dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__);
26892815 		/* Unable to map in GPU space. */
26902816 		WARN_ON(1);
....@@ -2709,7 +2835,7 @@
27092835 	vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT);
27102836 out:
27112837 	*regm = reg;
2712
-	dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n");
2838
+	dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
27132839 
27142840 	return err;
27152841 }
....@@ -2750,17 +2876,10 @@
27502876 		goto out_unlock;
27512877 	}
27522878 
2753
-	/* if not the MTP, verify that the MTP has been mapped */
2754
-	rcu_read_lock();
2755
-	/* catches both when the special page isn't present or
2756
-	 * when we've forked
2757
-	 */
2758
-	if (rcu_dereference(kctx->process_mm) != current->mm) {
2879
+	if (!kbase_mem_allow_alloc(kctx)) {
27592880 		err = -EINVAL;
2760
-		rcu_read_unlock();
27612881 		goto out_unlock;
27622882 	}
2763
-	rcu_read_unlock();
27642883 
27652884 	switch (vma->vm_pgoff) {
27662885 	case PFN_DOWN(BASEP_MEM_INVALID_HANDLE):
....@@ -2769,6 +2888,7 @@
27692888 		err = -EINVAL;
27702889 		goto out_unlock;
27712890 	case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE):
2891
+#if defined(CONFIG_MALI_VECTOR_DUMP)
27722892 		/* MMU dump */
27732893 		err = kbase_mmu_dump_mmap(kctx, vma, &reg, &kaddr);
27742894 		if (err != 0)
....@@ -2776,6 +2896,11 @@
27762896 		/* free the region on munmap */
27772897 		free_on_close = 1;
27782898 		break;
2899
+#else
2900
+		/* Illegal handle for direct map */
2901
+		err = -EINVAL;
2902
+		goto out_unlock;
2903
+#endif /* defined(CONFIG_MALI_VECTOR_DUMP) */
27792904 #if MALI_USE_CSF
27802905 	case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE):
27812906 		kbase_gpu_vm_unlock(kctx);
....@@ -2846,8 +2971,7 @@
28462971 					dev_warn(dev, "mmap aliased: invalid params!\n");
28472972 					goto out_unlock;
28482973 				}
2849
-			}
2850
-			else if (reg->cpu_alloc->nents <
2974
+			} else if (reg->cpu_alloc->nents <
28512975 					(vma->vm_pgoff - reg->start_pfn + nr_pages)) {
28522976 				/* limit what we map to the amount currently backed */
28532977 				if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
....@@ -2864,7 +2988,7 @@
28642988 
28652989 	err = kbase_cpu_mmap(kctx, reg, vma, kaddr, nr_pages, aligned_offset,
28662990 			free_on_close);
2867
-
2991
+#if defined(CONFIG_MALI_VECTOR_DUMP)
28682992 	if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) {
28692993 		/* MMU dump - userspace should now have a reference on
28702994 		 * the pages, so we can now free the kernel mapping
....@@ -2883,7 +3007,7 @@
28833007 		 */
28843008 		vma->vm_pgoff = PFN_DOWN(vma->vm_start);
28853009 	}
2886
-
3010
+#endif /* defined(CONFIG_MALI_VECTOR_DUMP) */
28873011 out_unlock:
28883012 	kbase_gpu_vm_unlock(kctx);
28893013 out:
....@@ -2925,9 +3049,102 @@
29253049 	}
29263050 }
29273051 
2928
-static int kbase_vmap_phy_pages(struct kbase_context *kctx,
2929
-		struct kbase_va_region *reg, u64 offset_bytes, size_t size,
2930
-		struct kbase_vmap_struct *map)
3052
+/**
3053
+ * kbase_vmap_phy_pages_migrate_count_increment - Increment VMAP count for
3054
+ *                                                array of physical pages
3055
+ *
3056
+ * @pages:      Array of pages.
3057
+ * @page_count: Number of pages.
3058
+ * @flags:      Region flags.
3059
+ *
3060
+ * This function is supposed to be called only if page migration support
3061
+ * is enabled in the driver.
3062
+ *
3063
+ * The counter of kernel CPU mappings of the physical pages involved in a
3064
+ * mapping operation is incremented by 1. Errors are handled by making pages
3065
+ * not movable. Permanent kernel mappings will be marked as not movable, too.
3066
+ */
3067
+static void kbase_vmap_phy_pages_migrate_count_increment(struct tagged_addr *pages,
3068
+							 size_t page_count, unsigned long flags)
3069
+{
3070
+	size_t i;
3071
+
3072
+	for (i = 0; i < page_count; i++) {
3073
+		struct page *p = as_page(pages[i]);
3074
+		struct kbase_page_metadata *page_md = kbase_page_private(p);
3075
+
3076
+		/* Skip the 4KB page that is part of a large page, as the large page is
3077
+		 * excluded from the migration process.
3078
+		 */
3079
+		if (is_huge(pages[i]) || is_partial(pages[i]))
3080
+			continue;
3081
+
3082
+		spin_lock(&page_md->migrate_lock);
3083
+		/* Mark permanent kernel mappings as NOT_MOVABLE because they're likely
3084
+		 * to stay mapped for a long time. However, keep on counting the number
3085
+		 * of mappings even for them: they don't represent an exception for the
3086
+		 * vmap_count.
3087
+		 *
3088
+		 * At the same time, errors need to be handled if a client tries to add
3089
+		 * too many mappings, hence a page may end up in the NOT_MOVABLE state
3090
+		 * anyway even if it's not a permanent kernel mapping.
3091
+		 */
3092
+		if (flags & KBASE_REG_PERMANENT_KERNEL_MAPPING)
3093
+			page_md->status = PAGE_STATUS_SET(page_md->status, (u8)NOT_MOVABLE);
3094
+		if (page_md->vmap_count < U8_MAX)
3095
+			page_md->vmap_count++;
3096
+		else
3097
+			page_md->status = PAGE_STATUS_SET(page_md->status, (u8)NOT_MOVABLE);
3098
+		spin_unlock(&page_md->migrate_lock);
3099
+	}
3100
+}
3101
+
3102
+/**
3103
+ * kbase_vunmap_phy_pages_migrate_count_decrement - Decrement VMAP count for
3104
+ *                                                  array of physical pages
3105
+ *
3106
+ * @pages:      Array of pages.
3107
+ * @page_count: Number of pages.
3108
+ *
3109
+ * This function is supposed to be called only if page migration support
3110
+ * is enabled in the driver.
3111
+ *
3112
+ * The counter of kernel CPU mappings of the physical pages involved in a
3113
+ * mapping operation is decremented by 1. Errors are handled by making pages
3114
+ * not movable.
3115
+ */
3116
+static void kbase_vunmap_phy_pages_migrate_count_decrement(struct tagged_addr *pages,
3117
+							   size_t page_count)
3118
+{
3119
+	size_t i;
3120
+
3121
+	for (i = 0; i < page_count; i++) {
3122
+		struct page *p = as_page(pages[i]);
3123
+		struct kbase_page_metadata *page_md = kbase_page_private(p);
3124
+
3125
+		/* Skip the 4KB page that is part of a large page, as the large page is
3126
+		 * excluded from the migration process.
3127
+		 */
3128
+		if (is_huge(pages[i]) || is_partial(pages[i]))
3129
+			continue;
3130
+
3131
+		spin_lock(&page_md->migrate_lock);
3132
+		/* Decrement the number of mappings for all kinds of pages, including
3133
+		 * pages which are NOT_MOVABLE (e.g. permanent kernel mappings).
3134
+		 * However, errors still need to be handled if a client tries to remove
3135
+		 * more mappings than created.
3136
+		 */
3137
+		if (page_md->vmap_count == 0)
3138
+			page_md->status = PAGE_STATUS_SET(page_md->status, (u8)NOT_MOVABLE);
3139
+		else
3140
+			page_md->vmap_count--;
3141
+		spin_unlock(&page_md->migrate_lock);
3142
+	}
3143
+}
3144
+
3145
+static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_region *reg,
3146
+				u64 offset_bytes, size_t size, struct kbase_vmap_struct *map,
3147
+				kbase_vmap_flag vmap_flags)
29313148 {
29323149 	unsigned long page_index;
29333150 	unsigned int offset_in_page = offset_bytes & ~PAGE_MASK;
....@@ -2937,6 +3154,12 @@
29373154 	void *cpu_addr = NULL;
29383155 	pgprot_t prot;
29393156 	size_t i;
3157
+
3158
+	if (WARN_ON(vmap_flags & ~KBASE_VMAP_INPUT_FLAGS))
3159
+		return -EINVAL;
3160
+
3161
+	if (WARN_ON(kbase_is_region_invalid_or_free(reg)))
3162
+		return -EINVAL;
29403163 
29413164 	if (!size || !map || !reg->cpu_alloc || !reg->gpu_alloc)
29423165 		return -EINVAL;
....@@ -2953,6 +3176,17 @@
29533176 
29543177 	if (page_index + page_count > kbase_reg_current_backed_size(reg))
29553178 		return -ENOMEM;
3179
+
3180
+	if ((vmap_flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) &&
3181
+	    (page_count > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES -
3182
+			   atomic_read(&kctx->permanent_mapped_pages)))) {
3183
+		dev_warn(
3184
+			kctx->kbdev->dev,
3185
+			"Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages",
3186
+			(u64)page_count, KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES,
3187
+			atomic_read(&kctx->permanent_mapped_pages));
3188
+		return -ENOMEM;
3189
+	}
29563190 
29573191 	if (reg->flags & KBASE_REG_DONT_NEED)
29583192 		return -EINVAL;
....@@ -2980,6 +3214,13 @@
29803214 	 */
29813215 	cpu_addr = vmap(pages, page_count, VM_MAP, prot);
29823216 
3217
+	/* If page migration is enabled, increment the number of VMA mappings
3218
+	 * of all physical pages. In case of errors, e.g. too many mappings,
3219
+	 * make the page not movable to prevent trouble.
3220
+	 */
3221
+	if (kbase_page_migration_enabled && !kbase_mem_is_imported(reg->gpu_alloc->type))
3222
+		kbase_vmap_phy_pages_migrate_count_increment(page_array, page_count, reg->flags);
3223
+
29833224 	kfree(pages);
29843225 
29853226 	if (!cpu_addr)
....@@ -2992,14 +3233,55 @@
29923233 	map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index];
29933234 	map->addr = (void *)((uintptr_t)cpu_addr + offset_in_page);
29943235 	map->size = size;
2995
-	map->sync_needed = ((reg->flags & KBASE_REG_CPU_CACHED) != 0) &&
2996
-		!kbase_mem_is_imported(map->gpu_alloc->type);
3236
+	map->flags = vmap_flags;
3237
+	if ((reg->flags & KBASE_REG_CPU_CACHED) && !kbase_mem_is_imported(map->gpu_alloc->type))
3238
+		map->flags |= KBASE_VMAP_FLAG_SYNC_NEEDED;
29973239 
2998
-	if (map->sync_needed)
3240
+	if (map->flags & KBASE_VMAP_FLAG_SYNC_NEEDED)
29993241 		kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU);
30003242 
3243
+	if (vmap_flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING)
3244
+		atomic_add(page_count, &kctx->permanent_mapped_pages);
3245
+
30013246 	kbase_mem_phy_alloc_kernel_mapped(reg->cpu_alloc);
3247
+
30023248 	return 0;
3249
+}
3250
+
3251
+void *kbase_vmap_reg(struct kbase_context *kctx, struct kbase_va_region *reg, u64 gpu_addr,
3252
+		     size_t size, unsigned long prot_request, struct kbase_vmap_struct *map,
3253
+		     kbase_vmap_flag vmap_flags)
3254
+{
3255
+	u64 offset_bytes;
3256
+	struct kbase_mem_phy_alloc *cpu_alloc;
3257
+	struct kbase_mem_phy_alloc *gpu_alloc;
3258
+	int err;
3259
+
3260
+	lockdep_assert_held(&kctx->reg_lock);
3261
+
3262
+	if (WARN_ON(kbase_is_region_invalid_or_free(reg)))
3263
+		return NULL;
3264
+
3265
+	/* check access permissions can be satisfied
3266
+	 * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR}
3267
+	 */
3268
+	if ((reg->flags & prot_request) != prot_request)
3269
+		return NULL;
3270
+
3271
+	offset_bytes = gpu_addr - (reg->start_pfn << PAGE_SHIFT);
3272
+	cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
3273
+	gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
3274
+
3275
+	err = kbase_vmap_phy_pages(kctx, reg, offset_bytes, size, map, vmap_flags);
3276
+	if (err < 0)
3277
+		goto fail_vmap_phy_pages;
3278
+
3279
+	return map->addr;
3280
+
3281
+fail_vmap_phy_pages:
3282
+	kbase_mem_phy_alloc_put(cpu_alloc);
3283
+	kbase_mem_phy_alloc_put(gpu_alloc);
3284
+	return NULL;
30033285 }
30043286 
30053287 void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size,
....@@ -3007,44 +3289,21 @@
30073289 {
30083290 	struct kbase_va_region *reg;
30093291 	void *addr = NULL;
3010
-	u64 offset_bytes;
3011
-	struct kbase_mem_phy_alloc *cpu_alloc;
3012
-	struct kbase_mem_phy_alloc *gpu_alloc;
3013
-	int err;
30143292 
30153293 	kbase_gpu_vm_lock(kctx);
30163294 
3017
-	reg = kbase_region_tracker_find_region_enclosing_address(kctx,
3018
-			gpu_addr);
3295
+	reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr);
30193296 	if (kbase_is_region_invalid_or_free(reg))
30203297 		goto out_unlock;
30213298 
3022
-	/* check access permissions can be satisfied
3023
-	 * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR}
3024
-	 */
3025
-	if ((reg->flags & prot_request) != prot_request)
3299
+	if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE)
30263300 		goto out_unlock;
30273301 
3028
-	offset_bytes = gpu_addr - (reg->start_pfn << PAGE_SHIFT);
3029
-	cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
3030
-	gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc);
3031
-
3032
-	err = kbase_vmap_phy_pages(kctx, reg, offset_bytes, size, map);
3033
-	if (err < 0)
3034
-		goto fail_vmap_phy_pages;
3035
-
3036
-	addr = map->addr;
3302
+	addr = kbase_vmap_reg(kctx, reg, gpu_addr, size, prot_request, map, 0u);
30373303 
30383304 out_unlock:
30393305 	kbase_gpu_vm_unlock(kctx);
30403306 	return addr;
3041
-
3042
-fail_vmap_phy_pages:
3043
-	kbase_gpu_vm_unlock(kctx);
3044
-	kbase_mem_phy_alloc_put(cpu_alloc);
3045
-	kbase_mem_phy_alloc_put(gpu_alloc);
3046
-
3047
-	return NULL;
30483307 }
30493308 
30503309 void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size,
....@@ -3064,18 +3323,37 @@
30643323 		struct kbase_vmap_struct *map)
30653324 {
30663325 	void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
3326
+
30673327 	vunmap(addr);
30683328 
3069
-	if (map->sync_needed)
3329
+	/* If page migration is enabled, decrement the number of VMA mappings
3330
+	 * for all physical pages. Now is a good time to do it because references
3331
+	 * haven't been released yet.
3332
+	 */
3333
+	if (kbase_page_migration_enabled && !kbase_mem_is_imported(map->gpu_alloc->type)) {
3334
+		const size_t page_count = PFN_UP(map->offset_in_page + map->size);
3335
+		struct tagged_addr *pages_array = map->cpu_pages;
3336
+
3337
+		kbase_vunmap_phy_pages_migrate_count_decrement(pages_array, page_count);
3338
+	}
3339
+
3340
+	if (map->flags & KBASE_VMAP_FLAG_SYNC_NEEDED)
30703341 		kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE);
3342
+	if (map->flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) {
3343
+		size_t page_count = PFN_UP(map->offset_in_page + map->size);
3344
+
3345
+		WARN_ON(page_count > atomic_read(&kctx->permanent_mapped_pages));
3346
+		atomic_sub(page_count, &kctx->permanent_mapped_pages);
3347
+	}
30713348 
30723349 	kbase_mem_phy_alloc_kernel_unmapped(map->cpu_alloc);
3350
+
30733351 	map->offset_in_page = 0;
30743352 	map->cpu_pages = NULL;
30753353 	map->gpu_pages = NULL;
30763354 	map->addr = NULL;
30773355 	map->size = 0;
3078
-	map->sync_needed = false;
3356
+	map->flags = 0;
30793357 }
30803358 
30813359 void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map)
....@@ -3102,79 +3380,29 @@
31023380 
31033381 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages)
31043382 {
3105
-	struct mm_struct *mm;
3383
+	struct mm_struct *mm = kctx->process_mm;
31063384 
3107
-	rcu_read_lock();
3108
-	mm = rcu_dereference(kctx->process_mm);
3109
-	if (mm) {
3110
-		atomic_add(pages, &kctx->nonmapped_pages);
3111
-#ifdef SPLIT_RSS_COUNTING
3112
-		kbasep_add_mm_counter(mm, MM_FILEPAGES, pages);
3113
-#else
3114
-		spin_lock(&mm->page_table_lock);
3115
-		kbasep_add_mm_counter(mm, MM_FILEPAGES, pages);
3116
-		spin_unlock(&mm->page_table_lock);
3117
-#endif
3118
-	}
3119
-	rcu_read_unlock();
3120
-}
3121
-
3122
-static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx)
3123
-{
3124
-	int pages;
3125
-	struct mm_struct *mm;
3126
-
3127
-	spin_lock(&kctx->mm_update_lock);
3128
-	mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock));
3129
-	if (!mm) {
3130
-		spin_unlock(&kctx->mm_update_lock);
3385
+	if (unlikely(!mm))
31313386 		return;
3132
-	}
31333387 
3134
-	rcu_assign_pointer(kctx->process_mm, NULL);
3135
-	spin_unlock(&kctx->mm_update_lock);
3136
-	synchronize_rcu();
3137
-
3138
-	pages = atomic_xchg(&kctx->nonmapped_pages, 0);
3388
+	atomic_add(pages, &kctx->nonmapped_pages);
31393389 #ifdef SPLIT_RSS_COUNTING
3140
-	kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages);
3390
+	kbasep_add_mm_counter(mm, MM_FILEPAGES, pages);
31413391 #else
31423392 	spin_lock(&mm->page_table_lock);
3143
-	kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages);
3393
+	kbasep_add_mm_counter(mm, MM_FILEPAGES, pages);
31443394 	spin_unlock(&mm->page_table_lock);
31453395 #endif
31463396 }
31473397 
3148
-static void kbase_special_vm_close(struct vm_area_struct *vma)
3149
-{
3150
-	struct kbase_context *kctx;
3151
-
3152
-	kctx = vma->vm_private_data;
3153
-	kbasep_os_process_page_usage_drain(kctx);
3154
-}
3155
-
3156
-static const struct vm_operations_struct kbase_vm_special_ops = {
3157
-	.close = kbase_special_vm_close,
3158
-};
3159
-
31603398 static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma)
31613399 {
3162
-	/* check that this is the only tracking page */
3163
-	spin_lock(&kctx->mm_update_lock);
3164
-	if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) {
3165
-		spin_unlock(&kctx->mm_update_lock);
3166
-		return -EFAULT;
3167
-	}
3168
-
3169
-	rcu_assign_pointer(kctx->process_mm, current->mm);
3170
-
3171
-	spin_unlock(&kctx->mm_update_lock);
3400
+	if (vma_pages(vma) != 1)
3401
+		return -EINVAL;
31723402 
31733403 	/* no real access */
31743404 	vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC);
31753405 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
3176
-	vma->vm_ops = &kbase_vm_special_ops;
3177
-	vma->vm_private_data = kctx;
31783406 
31793407 	return 0;
31803408 }
....@@ -3189,15 +3417,37 @@
31893417 	 * assigned one, otherwise a dummy page. Always return the
31903418 	 * dummy page in no mali builds.
31913419 	 */
3420
+#if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)
3421
+	return PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_db_page));
3422
+#else
31923423 	if (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID)
31933424 		return PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_db_page));
3425
+#endif
31943426 	return (PFN_DOWN(kbdev->reg_start + CSF_HW_DOORBELL_PAGE_OFFSET +
31953427 			 (u64)queue->doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE));
31963428 }
31973429 
3430
+static int
3431
+#if (KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE || \
3432
+	KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE)
3433
+kbase_csf_user_io_pages_vm_mremap(struct vm_area_struct *vma)
3434
+#else
3435
+kbase_csf_user_io_pages_vm_mremap(struct vm_area_struct *vma, unsigned long flags)
3436
+#endif
3437
+{
3438
+	pr_debug("Unexpected call to mremap method for User IO pages mapping vma\n");
3439
+	return -EINVAL;
3440
+}
3441
+
3442
+static int kbase_csf_user_io_pages_vm_split(struct vm_area_struct *vma, unsigned long addr)
3443
+{
3444
+	pr_debug("Unexpected call to split method for User IO pages mapping vma\n");
3445
+	return -EINVAL;
3446
+}
3447
+
31983448 static void kbase_csf_user_io_pages_vm_open(struct vm_area_struct *vma)
31993449 {
3200
-	WARN(1, "Unexpected attempt to clone private vma\n");
3450
+	pr_debug("Unexpected call to the open method for User IO pages mapping vma\n");
32013451 	vma->vm_private_data = NULL;
32023452 }
32033453 
....@@ -3209,8 +3459,10 @@
32093459 	int err;
32103460 	bool reset_prevented = false;
32113461 
3212
-	if (WARN_ON(!queue))
3462
+	if (!queue) {
3463
+		pr_debug("Close method called for the new User IO pages mapping vma\n");
32133464 		return;
3465
+	}
32143466 
32153467 	kctx = queue->kctx;
32163468 	kbdev = kctx->kbdev;
....@@ -3225,7 +3477,7 @@
32253477 		reset_prevented = true;
32263478 
32273479 	mutex_lock(&kctx->csf.lock);
3228
-	kbase_csf_queue_unbind(queue);
3480
+	kbase_csf_queue_unbind(queue, is_process_exiting(vma));
32293481 	mutex_unlock(&kctx->csf.lock);
32303482 
32313483 	if (reset_prevented)
....@@ -3254,24 +3506,21 @@
32543506 	struct memory_group_manager_device *mgm_dev;
32553507 
32563508 	/* Few sanity checks up front */
3257
-	if ((nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) ||
3258
-	    (vma->vm_pgoff != queue->db_file_offset))
3509
+	if (!queue || (nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) ||
3510
+	    (vma->vm_pgoff != queue->db_file_offset)) {
3511
+		pr_warn("Unexpected CPU page fault on User IO pages mapping for process %s tgid %d pid %d\n",
3512
+			current->comm, current->tgid, current->pid);
32593513 		return VM_FAULT_SIGBUS;
3514
+	}
32603515 
3261
-	mutex_lock(&queue->kctx->csf.lock);
32623516 	kbdev = queue->kctx->kbdev;
32633517 	mgm_dev = kbdev->mgm_dev;
3518
+
3519
+	mutex_lock(&kbdev->csf.reg_lock);
32643520 
32653521 	/* Always map the doorbell page as uncached */
32663522 	doorbell_pgprot = pgprot_device(vma->vm_page_prot);
32673523 
3268
-#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \
3269
-		((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \
3270
-		 (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE)))
3271
-	vma->vm_page_prot = doorbell_pgprot;
3272
-	input_page_pgprot = doorbell_pgprot;
3273
-	output_page_pgprot = doorbell_pgprot;
3274
-#else
32753524 	if (kbdev->system_coherency == COHERENCY_NONE) {
32763525 		input_page_pgprot = pgprot_writecombine(vma->vm_page_prot);
32773526 		output_page_pgprot = pgprot_writecombine(vma->vm_page_prot);
....@@ -3279,7 +3528,6 @@
32793528 		input_page_pgprot = vma->vm_page_prot;
32803529 		output_page_pgprot = vma->vm_page_prot;
32813530 	}
3282
-#endif
32833531 
32843532 	doorbell_cpu_addr = vma->vm_start;
32853533 
....@@ -3288,12 +3536,10 @@
32883536 #else
32893537 	if (vmf->address == doorbell_cpu_addr) {
32903538 #endif
3291
-		mutex_lock(&kbdev->csf.reg_lock);
32923539 		doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue);
32933540 		ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev,
32943541 			KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr,
32953542 			doorbell_page_pfn, doorbell_pgprot);
3296
-		mutex_unlock(&kbdev->csf.reg_lock);
32973543 	} else {
32983544 		/* Map the Input page */
32993545 		input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE;
....@@ -3313,13 +3559,19 @@
33133559 	}
33143560 
33153561 exit:
3316
-	mutex_unlock(&queue->kctx->csf.lock);
3562
+	mutex_unlock(&kbdev->csf.reg_lock);
33173563 	return ret;
33183564 }
33193565 
33203566 static const struct vm_operations_struct kbase_csf_user_io_pages_vm_ops = {
33213567 	.open = kbase_csf_user_io_pages_vm_open,
33223568 	.close = kbase_csf_user_io_pages_vm_close,
3569
+#if KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE
3570
+	.may_split = kbase_csf_user_io_pages_vm_split,
3571
+#else
3572
+	.split = kbase_csf_user_io_pages_vm_split,
3573
+#endif
3574
+	.mremap = kbase_csf_user_io_pages_vm_mremap,
33233575 	.fault = kbase_csf_user_io_pages_vm_fault
33243576 };
33253577 
....@@ -3399,13 +3651,75 @@
33993651 	return err;
34003652 }
34013653 
3654
+/**
3655
+ * kbase_csf_user_reg_vm_open - VMA open function for the USER page
3656
+ *
3657
+ * @vma:  Pointer to the struct containing information about
3658
+ *        the userspace mapping of USER page.
3659
+ * Note:
3660
+ * This function isn't expected to be called. If called (i.e> mremap),
3661
+ * set private_data as NULL to indicate to close() and fault() functions.
3662
+ */
3663
+static void kbase_csf_user_reg_vm_open(struct vm_area_struct *vma)
3664
+{
3665
+	pr_debug("Unexpected call to the open method for USER register mapping");
3666
+	vma->vm_private_data = NULL;
3667
+}
3668
+
3669
+/**
3670
+ * kbase_csf_user_reg_vm_close - VMA close function for the USER page
3671
+ *
3672
+ * @vma:  Pointer to the struct containing information about
3673
+ *        the userspace mapping of USER page.
3674
+ */
34023675 static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma)
34033676 {
34043677 	struct kbase_context *kctx = vma->vm_private_data;
3678
+	struct kbase_device *kbdev;
34053679 
3406
-	WARN_ON(!kctx->csf.user_reg_vma);
3680
+	if (unlikely(!kctx)) {
3681
+		pr_debug("Close function called for the unexpected mapping");
3682
+		return;
3683
+	}
34073684 
3408
-	kctx->csf.user_reg_vma = NULL;
3685
+	kbdev = kctx->kbdev;
3686
+
3687
+	if (unlikely(!kctx->csf.user_reg.vma))
3688
+		dev_warn(kbdev->dev, "user_reg VMA pointer unexpectedly NULL for ctx %d_%d",
3689
+			 kctx->tgid, kctx->id);
3690
+
3691
+	mutex_lock(&kbdev->csf.reg_lock);
3692
+	list_del_init(&kctx->csf.user_reg.link);
3693
+	mutex_unlock(&kbdev->csf.reg_lock);
3694
+
3695
+	kctx->csf.user_reg.vma = NULL;
3696
+
3697
+	/* Now as the VMA is closed, drop the reference on mali device file */
3698
+	fput(kctx->filp);
3699
+}
3700
+
3701
+/**
3702
+ * kbase_csf_user_reg_vm_mremap - VMA mremap function for the USER page
3703
+ *
3704
+ * @vma:  Pointer to the struct containing information about
3705
+ *        the userspace mapping of USER page.
3706
+ *
3707
+ * Return: -EINVAL
3708
+ *
3709
+ * Note:
3710
+ * User space must not attempt mremap on USER page mapping.
3711
+ * This function will return an error to fail the attempt.
3712
+ */
3713
+static int
3714
+#if ((KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE) || \
3715
+	(KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE))
3716
+kbase_csf_user_reg_vm_mremap(struct vm_area_struct *vma)
3717
+#else
3718
+kbase_csf_user_reg_vm_mremap(struct vm_area_struct *vma, unsigned long flags)
3719
+#endif
3720
+{
3721
+	pr_debug("Unexpected call to mremap method for USER page mapping vma\n");
3722
+	return -EINVAL;
34093723 }
34103724 
34113725 #if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE)
....@@ -3418,39 +3732,52 @@
34183732 	struct vm_area_struct *vma = vmf->vma;
34193733 #endif
34203734 	struct kbase_context *kctx = vma->vm_private_data;
3421
-	struct kbase_device *kbdev = kctx->kbdev;
3422
-	struct memory_group_manager_device *mgm_dev = kbdev->mgm_dev;
3423
-	unsigned long pfn = PFN_DOWN(kbdev->reg_start + USER_BASE);
3735
+	struct kbase_device *kbdev;
3736
+	struct memory_group_manager_device *mgm_dev;
3737
+	unsigned long pfn;
34243738 	size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start);
34253739 	vm_fault_t ret = VM_FAULT_SIGBUS;
3740
+	unsigned long flags;
34263741 
34273742 	/* Few sanity checks up front */
3428
-	if (WARN_ON(nr_pages != 1) ||
3429
-	    WARN_ON(vma != kctx->csf.user_reg_vma) ||
3430
-	    WARN_ON(vma->vm_pgoff !=
3431
-			PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE)))
3743
+
3744
+	if (!kctx || (nr_pages != 1) || (vma != kctx->csf.user_reg.vma) ||
3745
+	    (vma->vm_pgoff != kctx->csf.user_reg.file_offset)) {
3746
+		pr_err("Unexpected CPU page fault on USER page mapping for process %s tgid %d pid %d\n",
3747
+		       current->comm, current->tgid, current->pid);
34323748 		return VM_FAULT_SIGBUS;
3749
+	}
34333750 
3434
-	mutex_lock(&kbdev->pm.lock);
3751
+	kbdev = kctx->kbdev;
3752
+	mgm_dev = kbdev->mgm_dev;
3753
+	pfn = PFN_DOWN(kbdev->reg_start + USER_BASE);
34353754 
3436
-	/* Don't map in the actual register page if GPU is powered down.
3437
-	 * Always map in the dummy page in no mali builds.
3755
+	mutex_lock(&kbdev->csf.reg_lock);
3756
+
3757
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
3758
+	/* Dummy page will be mapped during GPU off.
3759
+	 *
3760
+	 * In no mail builds, always map in the dummy page.
34383761 	 */
3439
-	if (!kbdev->pm.backend.gpu_powered)
3440
-		pfn = PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_user_reg_page));
3762
+	if (IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) || !kbdev->pm.backend.gpu_powered)
3763
+		pfn = PFN_DOWN(as_phys_addr_t(kbdev->csf.user_reg.dummy_page));
3764
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
34413765 
3766
+	list_move_tail(&kctx->csf.user_reg.link, &kbdev->csf.user_reg.list);
34423767 	ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev,
34433768 						   KBASE_MEM_GROUP_CSF_FW, vma,
34443769 						   vma->vm_start, pfn,
34453770 						   vma->vm_page_prot);
34463771 
3447
-	mutex_unlock(&kbdev->pm.lock);
3772
+	mutex_unlock(&kbdev->csf.reg_lock);
34483773 
34493774 	return ret;
34503775 }
34513776 
34523777 static const struct vm_operations_struct kbase_csf_user_reg_vm_ops = {
3778
+	.open = kbase_csf_user_reg_vm_open,
34533779 	.close = kbase_csf_user_reg_vm_close,
3780
+	.mremap = kbase_csf_user_reg_vm_mremap,
34543781 	.fault = kbase_csf_user_reg_vm_fault
34553782 };
34563783 
....@@ -3458,9 +3785,10 @@
34583785 				struct vm_area_struct *vma)
34593786 {
34603787 	size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start);
3788
+	struct kbase_device *kbdev = kctx->kbdev;
34613789 
34623790 	/* Few sanity checks */
3463
-	if (kctx->csf.user_reg_vma)
3791
+	if (kctx->csf.user_reg.vma)
34643792 		return -EBUSY;
34653793 
34663794 	if (nr_pages != 1)
....@@ -3479,8 +3807,21 @@
34793807 	 */
34803808 	vma->vm_flags |= VM_PFNMAP;
34813809 
3482
-	kctx->csf.user_reg_vma = vma;
3810
+	kctx->csf.user_reg.vma = vma;
34833811 
3812
+	mutex_lock(&kbdev->csf.reg_lock);
3813
+	kctx->csf.user_reg.file_offset = kbdev->csf.user_reg.file_offset++;
3814
+	mutex_unlock(&kbdev->csf.reg_lock);
3815
+
3816
+	/* Make VMA point to the special internal file, but don't drop the
3817
+	 * reference on mali device file (that would be done later when the
3818
+	 * VMA is closed).
3819
+	 */
3820
+	vma->vm_file = kctx->kbdev->csf.user_reg.filp;
3821
+	get_file(vma->vm_file);
3822
+
3823
+	/* Also adjust the vm_pgoff */
3824
+	vma->vm_pgoff = kctx->csf.user_reg.file_offset;
34843825 	vma->vm_ops = &kbase_csf_user_reg_vm_ops;
34853826 	vma->vm_private_data = kctx;
34863827