forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-06 08f87f769b595151be1afeff53e144f543faa614 
 kernel/drivers/gpu/arm/bifrost/mali_kbase_mem.h
....@@ -1,7 +1,7 @@
11 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
22 /*
33  *
4
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
4
+ * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
55  *
66  * This program is free software and is provided to you under the terms of the
77  * GNU General Public License version 2 as published by the Free Software
....@@ -37,6 +37,8 @@
3737 #include "mali_kbase_defs.h"
3838 /* Required for kbase_mem_evictable_unmake */
3939 #include "mali_kbase_mem_linux.h"
40
+#include "mali_kbase_mem_migrate.h"
41
+#include "mali_kbase_refcount_defs.h"
4042 
4143 static inline void kbase_process_page_usage_inc(struct kbase_context *kctx,
4244 		int pages);
....@@ -182,6 +184,106 @@
182184 	} imported;
183185 };
184186 
187
+/**
188
+ * enum kbase_page_status - Status of a page used for page migration.
189
+ *
190
+ * @MEM_POOL: Stable state. Page is located in a memory pool and can safely
191
+ *            be migrated.
192
+ * @ALLOCATE_IN_PROGRESS: Transitory state. A page is set to this status as
193
+ *                        soon as it leaves a memory pool.
194
+ * @SPILL_IN_PROGRESS: Transitory state. Corner case where pages in a memory
195
+ *                     pool of a dying context are being moved to the device
196
+ *                     memory pool.
197
+ * @NOT_MOVABLE: Stable state. Page has been allocated for an object that is
198
+ *               not movable, but may return to be movable when the object
199
+ *               is freed.
200
+ * @ALLOCATED_MAPPED: Stable state. Page has been allocated, mapped to GPU
201
+ *                    and has reference to kbase_mem_phy_alloc object.
202
+ * @PT_MAPPED: Stable state. Similar to ALLOCATED_MAPPED, but page doesn't
203
+ *             reference kbase_mem_phy_alloc object. Used as a page in MMU
204
+ *             page table.
205
+ * @FREE_IN_PROGRESS: Transitory state. A page is set to this status as soon as
206
+ *                    the driver manages to acquire a lock on the page while
207
+ *                    unmapping it. This status means that a memory release is
208
+ *                    happening and it's still not complete.
209
+ * @FREE_ISOLATED_IN_PROGRESS: Transitory state. This is a very particular corner case.
210
+ *                             A page is isolated while it is in ALLOCATED_MAPPED state,
211
+ *                             but then the driver tries to destroy the allocation.
212
+ * @FREE_PT_ISOLATED_IN_PROGRESS: Transitory state. This is a very particular corner case.
213
+ *                                A page is isolated while it is in PT_MAPPED state, but
214
+ *                                then the driver tries to destroy the allocation.
215
+ *
216
+ * Pages can only be migrated in stable states.
217
+ */
218
+enum kbase_page_status {
219
+	MEM_POOL = 0,
220
+	ALLOCATE_IN_PROGRESS,
221
+	SPILL_IN_PROGRESS,
222
+	NOT_MOVABLE,
223
+	ALLOCATED_MAPPED,
224
+	PT_MAPPED,
225
+	FREE_IN_PROGRESS,
226
+	FREE_ISOLATED_IN_PROGRESS,
227
+	FREE_PT_ISOLATED_IN_PROGRESS,
228
+};
229
+
230
+#define PGD_VPFN_LEVEL_MASK ((u64)0x3)
231
+#define PGD_VPFN_LEVEL_GET_LEVEL(pgd_vpfn_level) (pgd_vpfn_level & PGD_VPFN_LEVEL_MASK)
232
+#define PGD_VPFN_LEVEL_GET_VPFN(pgd_vpfn_level) (pgd_vpfn_level & ~PGD_VPFN_LEVEL_MASK)
233
+#define PGD_VPFN_LEVEL_SET(pgd_vpfn, level)                                                        \
234
+	((pgd_vpfn & ~PGD_VPFN_LEVEL_MASK) | (level & PGD_VPFN_LEVEL_MASK))
235
+
236
+/**
237
+ * struct kbase_page_metadata - Metadata for each page in kbase
238
+ *
239
+ * @kbdev:         Pointer to kbase device.
240
+ * @dma_addr:      DMA address mapped to page.
241
+ * @migrate_lock:  A spinlock to protect the private metadata.
242
+ * @data:          Member in union valid based on @status.
243
+ * @status:        Status to keep track if page can be migrated at any
244
+ *                 given moment. MSB will indicate if page is isolated.
245
+ *                 Protected by @migrate_lock.
246
+ * @vmap_count:    Counter of kernel mappings.
247
+ * @group_id:      Memory group ID obtained at the time of page allocation.
248
+ *
249
+ * Each 4KB page will have a reference to this struct in the private field.
250
+ * This will be used to keep track of information required for Linux page
251
+ * migration functionality as well as address for DMA mapping.
252
+ */
253
+struct kbase_page_metadata {
254
+	dma_addr_t dma_addr;
255
+	spinlock_t migrate_lock;
256
+
257
+	union {
258
+		struct {
259
+			struct kbase_mem_pool *pool;
260
+			/* Pool could be terminated after page is isolated and therefore
261
+			 * won't be able to get reference to kbase device.
262
+			 */
263
+			struct kbase_device *kbdev;
264
+		} mem_pool;
265
+		struct {
266
+			struct kbase_va_region *reg;
267
+			struct kbase_mmu_table *mmut;
268
+			u64 vpfn;
269
+		} mapped;
270
+		struct {
271
+			struct kbase_mmu_table *mmut;
272
+			u64 pgd_vpfn_level;
273
+		} pt_mapped;
274
+		struct {
275
+			struct kbase_device *kbdev;
276
+		} free_isolated;
277
+		struct {
278
+			struct kbase_device *kbdev;
279
+		} free_pt_isolated;
280
+	} data;
281
+
282
+	u8 status;
283
+	u8 vmap_count;
284
+	u8 group_id;
285
+};
286
+
185287 /* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is
186288  * used to signify that a buffer was pinned when it was imported. Since the
187289  * reference count is limited by the number of atoms that can be submitted at
....@@ -204,6 +306,20 @@
204306 	KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0)
205307 };
206308 
309
+/**
310
+ * kbase_set_phy_alloc_page_status - Set the page migration status of the underlying
311
+ *                                   physical allocation.
312
+ * @alloc:  the physical allocation containing the pages whose metadata is going
313
+ *          to be modified
314
+ * @status: the status the pages should end up in
315
+ *
316
+ * Note that this function does not go through all of the checking to ensure that
317
+ * proper states are set. Instead, it is only used when we change the allocation
318
+ * to NOT_MOVABLE or from NOT_MOVABLE to ALLOCATED_MAPPED
319
+ */
320
+void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc,
321
+				     enum kbase_page_status status);
322
+
207323 static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc)
208324 {
209325 	KBASE_DEBUG_ASSERT(alloc);
....@@ -224,8 +340,9 @@
224340 }
225341 
226342 /**
227
- * kbase_mem_phy_alloc_kernel_mapped - Increment kernel_mappings
228
- * counter for a memory region to prevent commit and flag changes
343
+ * kbase_mem_phy_alloc_kernel_mapped - Increment kernel_mappings counter for a
344
+ *                                     memory region to prevent commit and flag
345
+ *                                     changes
229346  *
230347  * @alloc:  Pointer to physical pages tracking object
231348  */
....@@ -287,6 +404,8 @@
287404  *          that triggered incremental rendering by growing too much.
288405  * @rbtree:          Backlink to the red-black tree of memory regions.
289406  * @start_pfn:       The Page Frame Number in GPU virtual address space.
407
+ * @user_data:       The address of GPU command queue when VA region represents
408
+ *                   a ring buffer.
290409  * @nr_pages:        The size of the region in pages.
291410  * @initial_commit:  Initial commit, for aligning the start address and
292411  *                   correctly growing KBASE_REG_TILER_ALIGN_TOP regions.
....@@ -301,6 +420,8 @@
301420  * @jit_usage_id: The last just-in-time memory usage ID for this region.
302421  * @jit_bin_id:   The just-in-time memory bin this region came from.
303422  * @va_refcnt:    Number of users of this region. Protected by reg_lock.
423
+ * @no_user_free_count:    Number of contexts that want to prevent the region
424
+ *                         from being freed by userspace.
304425  * @heap_info_gpu_addr: Pointer to an object in GPU memory defining an end of
305426  *                      an allocated region
306427  *                      The object can be one of:
....@@ -324,6 +445,7 @@
324445 	struct list_head link;
325446 	struct rb_root *rbtree;
326447 	u64 start_pfn;
448
+	void *user_data;
327449 	size_t nr_pages;
328450 	size_t initial_commit;
329451 	size_t threshold_pages;
....@@ -356,19 +478,26 @@
356478 /* inner & outer shareable coherency */
357479 #define KBASE_REG_SHARE_BOTH        (1ul << 10)
358480 
481
+#if MALI_USE_CSF
482
+/* Space for 8 different zones */
483
+#define KBASE_REG_ZONE_BITS 3
484
+#else
359485 /* Space for 4 different zones */
360
-#define KBASE_REG_ZONE_MASK         ((KBASE_REG_ZONE_MAX - 1ul) << 11)
361
-#define KBASE_REG_ZONE(x)           (((x) & (KBASE_REG_ZONE_MAX - 1ul)) << 11)
486
+#define KBASE_REG_ZONE_BITS 2
487
+#endif
488
+
489
+#define KBASE_REG_ZONE_MASK (((1 << KBASE_REG_ZONE_BITS) - 1ul) << 11)
490
+#define KBASE_REG_ZONE(x) (((x) & ((1 << KBASE_REG_ZONE_BITS) - 1ul)) << 11)
362491 #define KBASE_REG_ZONE_IDX(x)       (((x) & KBASE_REG_ZONE_MASK) >> 11)
363492 
364
-#if ((KBASE_REG_ZONE_MAX - 1) & 0x3) != (KBASE_REG_ZONE_MAX - 1)
365
-#error KBASE_REG_ZONE_MAX too large for allocation of KBASE_REG_<...> bits
493
+#if KBASE_REG_ZONE_MAX > (1 << KBASE_REG_ZONE_BITS)
494
+#error "Too many zones for the number of zone bits defined"
366495 #endif
367496 
368497 /* GPU read access */
369
-#define KBASE_REG_GPU_RD            (1ul<<13)
498
+#define KBASE_REG_GPU_RD (1ul << 14)
370499 /* CPU read access */
371
-#define KBASE_REG_CPU_RD            (1ul<<14)
500
+#define KBASE_REG_CPU_RD (1ul << 15)
372501 
373502 /* Index of chosen MEMATTR for this region (0..7) */
374503 #define KBASE_REG_MEMATTR_MASK      (7ul << 16)
....@@ -377,6 +506,13 @@
377506 
378507 #define KBASE_REG_PROTECTED         (1ul << 19)
379508 
509
+/* Region belongs to a shrinker.
510
+ *
511
+ * This can either mean that it is part of the JIT/Ephemeral or tiler heap
512
+ * shrinker paths. Should be removed only after making sure that there are
513
+ * no references remaining to it in these paths, as it may cause the physical
514
+ * backing of the region to disappear during use.
515
+ */
380516 #define KBASE_REG_DONT_NEED         (1ul << 20)
381517 
382518 /* Imported buffer is padded? */
....@@ -406,10 +542,7 @@
406542 #define KBASE_REG_RESERVED_BIT_23   (1ul << 23)
407543 #endif /* !MALI_USE_CSF */
408544 
409
-/* Whilst this flag is set the GPU allocation is not supposed to be freed by
410
- * user space. The flag will remain set for the lifetime of JIT allocations.
411
- */
412
-#define KBASE_REG_NO_USER_FREE      (1ul << 24)
545
+/* Bit 24 is currently unused and is available for use for a new flag */
413546 
414547 /* Memory has permanent kernel side mapping */
415548 #define KBASE_REG_PERMANENT_KERNEL_MAPPING (1ul << 25)
....@@ -439,21 +572,39 @@
439572 /* Allocation is actively used for JIT memory */
440573 #define KBASE_REG_ACTIVE_JIT_ALLOC (1ul << 28)
441574 
442
-#define KBASE_REG_ZONE_SAME_VA      KBASE_REG_ZONE(0)
443
-
444
-/* only used with 32-bit clients */
445
-/*
446
- * On a 32bit platform, custom VA should be wired from 4GB
447
- * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface
448
- * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference).
449
- * So we put the default limit to the maximum possible on Linux and shrink
450
- * it down, if required by the GPU, during initialization.
575
+#if MALI_USE_CSF
576
+/* This flag only applies to allocations in the EXEC_FIXED_VA and FIXED_VA
577
+ * memory zones, and it determines whether they were created with a fixed
578
+ * GPU VA address requested by the user.
451579  */
580
+#define KBASE_REG_FIXED_ADDRESS (1ul << 29)
581
+#else
582
+#define KBASE_REG_RESERVED_BIT_29 (1ul << 29)
583
+#endif
584
+
585
+#define KBASE_REG_ZONE_SAME_VA      KBASE_REG_ZONE(0)
452586 
453587 #define KBASE_REG_ZONE_CUSTOM_VA         KBASE_REG_ZONE(1)
454588 #define KBASE_REG_ZONE_CUSTOM_VA_BASE    (0x100000000ULL >> PAGE_SHIFT)
455
-#define KBASE_REG_ZONE_CUSTOM_VA_SIZE    (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
589
+
590
+#if MALI_USE_CSF
591
+/* only used with 32-bit clients */
592
+/* On a 32bit platform, custom VA should be wired from 4GB to 2^(43).
593
+ */
594
+#define KBASE_REG_ZONE_CUSTOM_VA_SIZE \
595
+		(((1ULL << 43) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
596
+#else
597
+/* only used with 32-bit clients */
598
+/* On a 32bit platform, custom VA should be wired from 4GB to the VA limit of the
599
+ * GPU. Unfortunately, the Linux mmap() interface limits us to 2^32 pages (2^44
600
+ * bytes, see mmap64 man page for reference).  So we put the default limit to the
601
+ * maximum possible on Linux and shrink it down, if required by the GPU, during
602
+ * initialization.
603
+ */
604
+#define KBASE_REG_ZONE_CUSTOM_VA_SIZE \
605
+		(((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
456606 /* end 32-bit clients only */
607
+#endif
457608 
458609 /* The starting address and size of the GPU-executable zone are dynamic
459610  * and depend on the platform and the number of pages requested by the
....@@ -467,6 +618,33 @@
467618 #define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT)
468619 #define KBASE_REG_ZONE_MCU_SHARED_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - \
469620 		KBASE_REG_ZONE_MCU_SHARED_BASE)
621
+
622
+/* For CSF GPUs, the EXEC_VA zone is always 4GB in size, and starts at 2^47 for 64-bit
623
+ * clients, and 2^43 for 32-bit clients.
624
+ */
625
+#define KBASE_REG_ZONE_EXEC_VA_BASE_64 ((1ULL << 47) >> PAGE_SHIFT)
626
+#define KBASE_REG_ZONE_EXEC_VA_BASE_32 ((1ULL << 43) >> PAGE_SHIFT)
627
+#define KBASE_REG_ZONE_EXEC_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES
628
+
629
+/* Executable zone supporting FIXED/FIXABLE allocations.
630
+ * It is always 4GB in size.
631
+ */
632
+
633
+#define KBASE_REG_ZONE_EXEC_FIXED_VA KBASE_REG_ZONE(4)
634
+#define KBASE_REG_ZONE_EXEC_FIXED_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES
635
+
636
+/* Non-executable zone supporting FIXED/FIXABLE allocations.
637
+ * It extends from (2^47) up to (2^48)-1, for 64-bit userspace clients, and from
638
+ * (2^43) up to (2^44)-1 for 32-bit userspace clients.
639
+ */
640
+#define KBASE_REG_ZONE_FIXED_VA KBASE_REG_ZONE(5)
641
+
642
+/* Again - 32-bit userspace cannot map addresses beyond 2^44, but 64-bit can - and so
643
+ * the end of the FIXED_VA zone for 64-bit clients is (2^48)-1.
644
+ */
645
+#define KBASE_REG_ZONE_FIXED_VA_END_64 ((1ULL << 48) >> PAGE_SHIFT)
646
+#define KBASE_REG_ZONE_FIXED_VA_END_32 ((1ULL << 44) >> PAGE_SHIFT)
647
+
470648 #endif
471649 
472650 	unsigned long flags;
....@@ -476,6 +654,7 @@
476654 	struct list_head jit_node;
477655 	u16 jit_usage_id;
478656 	u8 jit_bin_id;
657
+
479658 #if MALI_JIT_PRESSURE_LIMIT_BASE
480659 	/* Pointer to an object in GPU memory defining an end of an allocated
481660 	 * region
....@@ -503,8 +682,26 @@
503682 	size_t used_pages;
504683 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
505684 
506
-	int    va_refcnt;
685
+	kbase_refcount_t va_refcnt;
686
+	atomic_t no_user_free_count;
507687 };
688
+
689
+/**
690
+ * kbase_is_ctx_reg_zone - determine whether a KBASE_REG_ZONE_<...> is for a
691
+ *                         context or for a device
692
+ * @zone_bits: A KBASE_REG_ZONE_<...> to query
693
+ *
694
+ * Return: True if the zone for @zone_bits is a context zone, False otherwise
695
+ */
696
+static inline bool kbase_is_ctx_reg_zone(unsigned long zone_bits)
697
+{
698
+	WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits);
699
+	return (zone_bits == KBASE_REG_ZONE_SAME_VA ||
700
+#if MALI_USE_CSF
701
+		zone_bits == KBASE_REG_ZONE_EXEC_FIXED_VA || zone_bits == KBASE_REG_ZONE_FIXED_VA ||
702
+#endif
703
+		zone_bits == KBASE_REG_ZONE_CUSTOM_VA || zone_bits == KBASE_REG_ZONE_EXEC_VA);
704
+}
508705 
509706 /* Special marker for failed JIT allocations that still must be marked as
510707  * in-use
....@@ -529,12 +726,31 @@
529726 	return (kbase_is_region_invalid(reg) ||	kbase_is_region_free(reg));
530727 }
531728 
532
-int kbase_remove_va_region(struct kbase_va_region *reg);
533
-static inline void kbase_region_refcnt_free(struct kbase_va_region *reg)
729
+/**
730
+ * kbase_is_region_shrinkable - Check if a region is "shrinkable".
731
+ * A shrinkable regions is a region for which its backing pages (reg->gpu_alloc->pages)
732
+ * can be freed at any point, even though the kbase_va_region structure itself
733
+ * may have been refcounted.
734
+ * Regions that aren't on a shrinker, but could be shrunk at any point in future
735
+ * without warning are still considered "shrinkable" (e.g. Active JIT allocs)
736
+ *
737
+ * @reg: Pointer to region
738
+ *
739
+ * Return: true if the region is "shrinkable", false if not.
740
+ */
741
+static inline bool kbase_is_region_shrinkable(struct kbase_va_region *reg)
742
+{
743
+	return (reg->flags & KBASE_REG_DONT_NEED) || (reg->flags & KBASE_REG_ACTIVE_JIT_ALLOC);
744
+}
745
+
746
+void kbase_remove_va_region(struct kbase_device *kbdev,
747
+			    struct kbase_va_region *reg);
748
+static inline void kbase_region_refcnt_free(struct kbase_device *kbdev,
749
+					    struct kbase_va_region *reg)
534750 {
535751 	/* If region was mapped then remove va region*/
536752 	if (reg->start_pfn)
537
-		kbase_remove_va_region(reg);
753
+		kbase_remove_va_region(kbdev, reg);
538754 
539755 	/* To detect use-after-free in debug builds */
540756 	KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE);
....@@ -544,14 +760,12 @@
544760 static inline struct kbase_va_region *kbase_va_region_alloc_get(
545761 		struct kbase_context *kctx, struct kbase_va_region *region)
546762 {
547
-	lockdep_assert_held(&kctx->reg_lock);
763
+	WARN_ON(!kbase_refcount_read(&region->va_refcnt));
764
+	WARN_ON(kbase_refcount_read(&region->va_refcnt) == INT_MAX);
548765 
549
-	WARN_ON(!region->va_refcnt);
550
-
551
-	/* non-atomic as kctx->reg_lock is held */
552766 	dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %pK\n",
553
-		region->va_refcnt, (void *)region);
554
-	region->va_refcnt++;
767
+		kbase_refcount_read(&region->va_refcnt), (void *)region);
768
+	kbase_refcount_inc(&region->va_refcnt);
555769 
556770 	return region;
557771 }
....@@ -559,19 +773,65 @@
559773 static inline struct kbase_va_region *kbase_va_region_alloc_put(
560774 		struct kbase_context *kctx, struct kbase_va_region *region)
561775 {
562
-	lockdep_assert_held(&kctx->reg_lock);
563
-
564
-	WARN_ON(region->va_refcnt <= 0);
776
+	WARN_ON(kbase_refcount_read(&region->va_refcnt) <= 0);
565777 	WARN_ON(region->flags & KBASE_REG_FREE);
566778 
567
-	/* non-atomic as kctx->reg_lock is held */
568
-	region->va_refcnt--;
569
-	dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %pK\n",
570
-		region->va_refcnt, (void *)region);
571
-	if (!region->va_refcnt)
572
-		kbase_region_refcnt_free(region);
779
+	if (kbase_refcount_dec_and_test(&region->va_refcnt))
780
+		kbase_region_refcnt_free(kctx->kbdev, region);
781
+	else
782
+		dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %pK\n",
783
+			kbase_refcount_read(&region->va_refcnt), (void *)region);
573784 
574785 	return NULL;
786
+}
787
+
788
+/**
789
+ * kbase_va_region_is_no_user_free - Check if user free is forbidden for the region.
790
+ * A region that must not be freed by userspace indicates that it is owned by some other
791
+ * kbase subsystem, for example tiler heaps, JIT memory or CSF queues.
792
+ * Such regions must not be shrunk (i.e. have their backing pages freed), except by the
793
+ * current owner.
794
+ * Hence, callers cannot rely on this check alone to determine if a region might be shrunk
795
+ * by any part of kbase. Instead they should use kbase_is_region_shrinkable().
796
+ *
797
+ * @region: Pointer to region.
798
+ *
799
+ * Return: true if userspace cannot free the region, false if userspace can free the region.
800
+ */
801
+static inline bool kbase_va_region_is_no_user_free(struct kbase_va_region *region)
802
+{
803
+	return atomic_read(&region->no_user_free_count) > 0;
804
+}
805
+
806
+/**
807
+ * kbase_va_region_no_user_free_inc - Increment "no user free" count for a region.
808
+ * Calling this function will prevent the region to be shrunk by parts of kbase that
809
+ * don't own the region (as long as the count stays above zero). Refer to
810
+ * kbase_va_region_is_no_user_free() for more information.
811
+ *
812
+ * @region: Pointer to region (not shrinkable).
813
+ *
814
+ * Return: the pointer to the region passed as argument.
815
+ */
816
+static inline void kbase_va_region_no_user_free_inc(struct kbase_va_region *region)
817
+{
818
+	WARN_ON(kbase_is_region_shrinkable(region));
819
+	WARN_ON(atomic_read(&region->no_user_free_count) == INT_MAX);
820
+
821
+	/* non-atomic as kctx->reg_lock is held */
822
+	atomic_inc(&region->no_user_free_count);
823
+}
824
+
825
+/**
826
+ * kbase_va_region_no_user_free_dec - Decrement "no user free" count for a region.
827
+ *
828
+ * @region: Pointer to region (not shrinkable).
829
+ */
830
+static inline void kbase_va_region_no_user_free_dec(struct kbase_va_region *region)
831
+{
832
+	WARN_ON(!kbase_va_region_is_no_user_free(region));
833
+
834
+	atomic_dec(&region->no_user_free_count);
575835 }
576836 
577837 /* Common functions */
....@@ -787,12 +1047,9 @@
7871047  *
7881048  * Return: 0 on success, negative -errno on error
7891049  */
790
-int kbase_mem_pool_init(struct kbase_mem_pool *pool,
791
-		const struct kbase_mem_pool_config *config,
792
-		unsigned int order,
793
-		int group_id,
794
-		struct kbase_device *kbdev,
795
-		struct kbase_mem_pool *next_pool);
1050
+int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config,
1051
+			unsigned int order, int group_id, struct kbase_device *kbdev,
1052
+			struct kbase_mem_pool *next_pool);
7961053 
7971054 /**
7981055  * kbase_mem_pool_term - Destroy a memory pool
....@@ -872,6 +1129,9 @@
8721129  * @pages:    Pointer to array where the physical address of the allocated
8731130  *            pages will be stored.
8741131  * @partial_allowed: If fewer pages allocated is allowed
1132
+ * @page_owner: Pointer to the task that created the Kbase context for which
1133
+ *              the pages are being allocated. It can be NULL if the pages
1134
+ *              won't be associated with any Kbase context.
8751135  *
8761136  * Like kbase_mem_pool_alloc() but optimized for allocating many pages.
8771137  *
....@@ -888,7 +1148,8 @@
8881148  * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead.
8891149  */
8901150 int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages,
891
-		struct tagged_addr *pages, bool partial_allowed);
1151
+			       struct tagged_addr *pages, bool partial_allowed,
1152
+			       struct task_struct *page_owner);
8921153 
8931154 /**
8941155  * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool
....@@ -1000,13 +1261,17 @@
10001261  * kbase_mem_pool_grow - Grow the pool
10011262  * @pool:       Memory pool to grow
10021263  * @nr_to_grow: Number of pages to add to the pool
1264
+ * @page_owner: Pointer to the task that created the Kbase context for which
1265
+ *              the memory pool is being grown. It can be NULL if the pages
1266
+ *              to be allocated won't be associated with any Kbase context.
10031267  *
10041268  * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to
10051269  * become larger than the maximum size specified.
10061270  *
1007
- * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages
1271
+ * Return: 0 on success, -ENOMEM if unable to allocate sufficent pages
10081272  */
1009
-int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow);
1273
+int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow,
1274
+			struct task_struct *page_owner);
10101275 
10111276 /**
10121277  * kbase_mem_pool_trim - Grow or shrink the pool to a new size
....@@ -1038,6 +1303,16 @@
10381303  * Return: A new page or NULL if no memory
10391304  */
10401305 struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool);
1306
+
1307
+/**
1308
+ * kbase_mem_pool_free_page - Free a page from a memory pool.
1309
+ * @pool:  Memory pool to free a page from
1310
+ * @p:     Page to free
1311
+ *
1312
+ * This will free any associated data stored for the page and release
1313
+ * the page back to the kernel.
1314
+ */
1315
+void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p);
10411316 
10421317 /**
10431318  * kbase_region_tracker_init - Initialize the region tracker data structure
....@@ -1086,9 +1361,9 @@
10861361 /**
10871362  * kbase_region_tracker_term_rbtree - Free memory for a region tracker
10881363  *
1089
- * This will free all the regions within the region tracker
1090
- *
10911364  * @rbtree: Region tracker tree root
1365
+ *
1366
+ * This will free all the regions within the region tracker
10921367  */
10931368 void kbase_region_tracker_term_rbtree(struct rb_root *rbtree);
10941369 
....@@ -1098,19 +1373,22 @@
10981373 		struct rb_root *rbtree, u64 gpu_addr);
10991374 
11001375 /**
1101
- * Check that a pointer is actually a valid region.
1376
+ * kbase_region_tracker_find_region_base_address - Check that a pointer is
1377
+ *                                                 actually a valid region.
11021378  * @kctx: kbase context containing the region
11031379  * @gpu_addr: pointer to check
11041380  *
11051381  * Must be called with context lock held.
1382
+ *
1383
+ * Return: pointer to the valid region on success, NULL otherwise
11061384  */
11071385 struct kbase_va_region *kbase_region_tracker_find_region_base_address(
11081386 		struct kbase_context *kctx, u64 gpu_addr);
11091387 struct kbase_va_region *kbase_find_region_base_address(struct rb_root *rbtree,
11101388 		u64 gpu_addr);
11111389 
1112
-struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree,
1113
-		u64 start_pfn, size_t nr_pages, int zone);
1390
+struct kbase_va_region *kbase_alloc_free_region(struct kbase_device *kbdev, struct rb_root *rbtree,
1391
+						u64 start_pfn, size_t nr_pages, int zone);
11141392 void kbase_free_alloced_region(struct kbase_va_region *reg);
11151393 int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg,
11161394 		u64 addr, size_t nr_pages, size_t align);
....@@ -1120,6 +1398,32 @@
11201398 
11211399 bool kbase_check_alloc_flags(unsigned long flags);
11221400 bool kbase_check_import_flags(unsigned long flags);
1401
+
1402
+static inline bool kbase_import_size_is_valid(struct kbase_device *kbdev, u64 va_pages)
1403
+{
1404
+	if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) {
1405
+		dev_dbg(
1406
+			kbdev->dev,
1407
+			"Import attempted with va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!",
1408
+			(unsigned long long)va_pages);
1409
+		return false;
1410
+	}
1411
+
1412
+	return true;
1413
+}
1414
+
1415
+static inline bool kbase_alias_size_is_valid(struct kbase_device *kbdev, u64 va_pages)
1416
+{
1417
+	if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) {
1418
+		dev_dbg(
1419
+			kbdev->dev,
1420
+			"Alias attempted with va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!",
1421
+			(unsigned long long)va_pages);
1422
+		return false;
1423
+	}
1424
+
1425
+	return true;
1426
+}
11231427 
11241428 /**
11251429  * kbase_check_alloc_sizes - check user space sizes parameters for an
....@@ -1155,29 +1459,86 @@
11551459 int kbase_update_region_flags(struct kbase_context *kctx,
11561460 		struct kbase_va_region *reg, unsigned long flags);
11571461 
1462
+/**
1463
+ * kbase_gpu_vm_lock() - Acquire the per-context region list lock
1464
+ * @kctx:  KBase context
1465
+ *
1466
+ * Care must be taken when making an allocation whilst holding this lock, because of interaction
1467
+ * with the Kernel's OoM-killer and use of this lock in &vm_operations_struct close() handlers.
1468
+ *
1469
+ * If this lock is taken during a syscall, and/or the allocation is 'small' then it is safe to use.
1470
+ *
1471
+ * If the caller is not in a syscall, and the allocation is 'large', then it must not hold this
1472
+ * lock.
1473
+ *
1474
+ * This is because the kernel OoM killer might target the process corresponding to that same kbase
1475
+ * context, and attempt to call the context's close() handlers for its open VMAs. This is safe if
1476
+ * the allocating caller is in a syscall, because the VMA close() handlers are delayed until all
1477
+ * syscalls have finished (noting that no new syscalls can start as the remaining user threads will
1478
+ * have been killed too), and so there is no possibility of contention between the thread
1479
+ * allocating with this lock held, and the VMA close() handler.
1480
+ *
1481
+ * However, outside of a syscall (e.g. a kworker or other kthread), one of kbase's VMA close()
1482
+ * handlers (kbase_cpu_vm_close()) also takes this lock, and so prevents the process from being
1483
+ * killed until the caller of the function allocating memory has released this lock. On subsequent
1484
+ * retries for allocating a page, the OoM killer would be re-invoked but skips over the process
1485
+ * stuck in its close() handler.
1486
+ *
1487
+ * Also because the caller is not in a syscall, the page allocation code in the kernel is not aware
1488
+ * that the allocation is being done on behalf of another process, and so does not realize that
1489
+ * process has received a kill signal due to an OoM, and so will continually retry with the OoM
1490
+ * killer until enough memory has been released, or until all other killable processes have been
1491
+ * killed (at which point the kernel halts with a panic).
1492
+ *
1493
+ * However, if the allocation outside of a syscall is small enough to be satisfied by killing
1494
+ * another process, then the allocation completes, the caller releases this lock, and
1495
+ * kbase_cpu_vm_close() can unblock and allow the process to be killed.
1496
+ *
1497
+ * Hence, this is effectively a deadlock with kbase_cpu_vm_close(), except that if the memory
1498
+ * allocation is small enough the deadlock can be resolved. For that reason, such a memory deadlock
1499
+ * is NOT discovered with CONFIG_PROVE_LOCKING.
1500
+ *
1501
+ * If this may be called outside of a syscall, consider moving allocations outside of this lock, or
1502
+ * use __GFP_NORETRY for such allocations (which will allow direct-reclaim attempts, but will
1503
+ * prevent OoM kills to satisfy the allocation, and will just fail the allocation instead).
1504
+ */
11581505 void kbase_gpu_vm_lock(struct kbase_context *kctx);
1506
+
1507
+/**
1508
+ * kbase_gpu_vm_unlock() - Release the per-context region list lock
1509
+ * @kctx:  KBase context
1510
+ */
11591511 void kbase_gpu_vm_unlock(struct kbase_context *kctx);
11601512 
11611513 int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size);
11621514 
11631515 /**
1164
- * Register region and map it on the GPU.
1516
+ * kbase_gpu_mmap - Register region and map it on the GPU.
1517
+ *
11651518  * @kctx: kbase context containing the region
11661519  * @reg: the region to add
11671520  * @addr: the address to insert the region at
11681521  * @nr_pages: the number of pages in the region
11691522  * @align: the minimum alignment in pages
1523
+ * @mmu_sync_info: Indicates whether this call is synchronous wrt MMU ops.
11701524  *
11711525  * Call kbase_add_va_region() and map the region on the GPU.
1526
+ *
1527
+ * Return: 0 on success, error code otherwise.
11721528  */
1173
-int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);
1529
+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg,
1530
+		   u64 addr, size_t nr_pages, size_t align,
1531
+		   enum kbase_caller_mmu_sync_info mmu_sync_info);
11741532 
11751533 /**
1176
- * Remove the region from the GPU and unregister it.
1534
+ * kbase_gpu_munmap - Remove the region from the GPU and unregister it.
1535
+ *
11771536  * @kctx:  KBase context
11781537  * @reg:   The region to remove
11791538  *
11801539  * Must be called with context lock held.
1540
+ *
1541
+ * Return: 0 on success, error code otherwise.
11811542  */
11821543 int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg);
11831544 
....@@ -1185,13 +1546,13 @@
11851546  * kbase_mmu_update - Configure an address space on the GPU to the specified
11861547  *                    MMU tables
11871548  *
1188
- * The caller has the following locking conditions:
1189
- * - It must hold kbase_device->mmu_hw_mutex
1190
- * - It must hold the hwaccess_lock
1191
- *
11921549  * @kbdev: Kbase device structure
11931550  * @mmut:  The set of MMU tables to be configured on the address space
11941551  * @as_nr: The address space to be configured
1552
+ *
1553
+ * The caller has the following locking conditions:
1554
+ * - It must hold kbase_device->mmu_hw_mutex
1555
+ * - It must hold the hwaccess_lock
11951556  */
11961557 void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut,
11971558 		int as_nr);
....@@ -1224,8 +1585,12 @@
12241585 
12251586 void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
12261587 
1588
+#if defined(CONFIG_MALI_VECTOR_DUMP)
12271589 /**
12281590  * kbase_mmu_dump() - Dump the MMU tables to a buffer.
1591
+ *
1592
+ * @kctx:        The kbase context to dump
1593
+ * @nr_pages:    The number of pages to allocate for the buffer.
12291594  *
12301595  * This function allocates a buffer (of @c nr_pages pages) to hold a dump
12311596  * of the MMU tables and fills it. If the buffer is too small
....@@ -1236,13 +1601,11 @@
12361601  * The buffer returned should be freed with @ref vfree when it is no longer
12371602  * required.
12381603  *
1239
- * @kctx:        The kbase context to dump
1240
- * @nr_pages:    The number of pages to allocate for the buffer.
1241
- *
12421604  * Return: The address of the buffer containing the MMU dump or NULL on error
12431605  * (including if the @c nr_pages is too small)
12441606  */
12451607 void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages);
1608
+#endif
12461609 
12471610 /**
12481611  * kbase_sync_now - Perform cache maintenance on a memory region
....@@ -1268,11 +1631,11 @@
12681631  * kbasep_os_process_page_usage_update() - Update the memory allocation
12691632  *                                         counters for the current process.
12701633  *
1271
- * OS specific call to updates the current memory allocation counters
1272
- * for the current process with the supplied delta.
1273
- *
12741634  * @kctx:  The kbase context
12751635  * @pages: The desired delta to apply to the memory usage counters.
1636
+ *
1637
+ * OS specific call to updates the current memory allocation counters
1638
+ * for the current process with the supplied delta.
12761639  */
12771640 
12781641 void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages);
....@@ -1281,11 +1644,11 @@
12811644  * kbase_process_page_usage_inc() - Add to the memory allocation counters for
12821645  *                                  the current process
12831646  *
1284
- * OS specific call to add to the current memory allocation counters for
1285
- * the current process by the supplied amount.
1286
- *
12871647  * @kctx:  The kernel base context used for the allocation.
12881648  * @pages: The desired delta to apply to the memory usage counters.
1649
+ *
1650
+ * OS specific call to add to the current memory allocation counters for
1651
+ * the current process by the supplied amount.
12891652  */
12901653 
12911654 static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages)
....@@ -1297,11 +1660,11 @@
12971660  * kbase_process_page_usage_dec() - Subtract from the memory allocation
12981661  *                                  counters for the current process.
12991662  *
1300
- * OS specific call to subtract from the current memory allocation counters
1301
- * for the current process by the supplied amount.
1302
- *
13031663  * @kctx:  The kernel base context used for the allocation.
13041664  * @pages: The desired delta to apply to the memory usage counters.
1665
+ *
1666
+ * OS specific call to subtract from the current memory allocation counters
1667
+ * for the current process by the supplied amount.
13051668  */
13061669 
13071670 static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages)
....@@ -1313,15 +1676,15 @@
13131676  * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU
13141677  * mapping of a memory allocation containing a given address range
13151678  *
1316
- * Searches for a CPU mapping of any part of any region that fully encloses the
1317
- * CPU virtual address range specified by @uaddr and @size. Returns a failure
1318
- * indication if only part of the address range lies within a CPU mapping.
1319
- *
13201679  * @kctx:      The kernel base context used for the allocation.
13211680  * @uaddr:     Start of the CPU virtual address range.
13221681  * @size:      Size of the CPU virtual address range (in bytes).
13231682  * @offset:    The offset from the start of the allocation to the specified CPU
13241683  *             virtual address.
1684
+ *
1685
+ * Searches for a CPU mapping of any part of any region that fully encloses the
1686
+ * CPU virtual address range specified by @uaddr and @size. Returns a failure
1687
+ * indication if only part of the address range lies within a CPU mapping.
13251688  *
13261689  * Return: 0 if offset was obtained successfully. Error code otherwise.
13271690  */
....@@ -1334,13 +1697,6 @@
13341697  * the start of GPU virtual memory region which encloses @gpu_addr for the
13351698  * @size length in bytes
13361699  *
1337
- * Searches for the memory region in GPU virtual memory space which contains
1338
- * the region defined by the @gpu_addr and @size, where @gpu_addr is the
1339
- * beginning and @size the length in bytes of the provided region. If found,
1340
- * the location of the start address of the GPU virtual memory region is
1341
- * passed in @start pointer and the location of the offset of the region into
1342
- * the GPU virtual memory region is passed in @offset pointer.
1343
- *
13441700  * @kctx:	The kernel base context within which the memory is searched.
13451701  * @gpu_addr:	GPU virtual address for which the region is sought; defines
13461702  *              the beginning of the provided region.
....@@ -1350,6 +1706,15 @@
13501706  *              the found GPU virtual memory region is.
13511707  * @offset:     Pointer to the location where the offset of @gpu_addr into
13521708  *              the found GPU virtual memory region is.
1709
+ *
1710
+ * Searches for the memory region in GPU virtual memory space which contains
1711
+ * the region defined by the @gpu_addr and @size, where @gpu_addr is the
1712
+ * beginning and @size the length in bytes of the provided region. If found,
1713
+ * the location of the start address of the GPU virtual memory region is
1714
+ * passed in @start pointer and the location of the offset of the region into
1715
+ * the GPU virtual memory region is passed in @offset pointer.
1716
+ *
1717
+ * Return: 0 on success, error code otherwise.
13531718  */
13541719 int kbasep_find_enclosing_gpu_mapping_start_and_offset(
13551720 		struct kbase_context *kctx,
....@@ -1360,15 +1725,21 @@
13601725  * @alloc:              allocation object to add pages to
13611726  * @nr_pages_requested: number of physical pages to allocate
13621727  *
1363
- * Allocates \a nr_pages_requested and updates the alloc object.
1728
+ * Allocates @nr_pages_requested and updates the alloc object.
13641729  *
1365
- * Return: 0 if all pages have been successfully allocated. Error code otherwise
1730
+ * Note: if kbase_gpu_vm_lock() is to be held around this function to ensure thread-safe updating
1731
+ * of @alloc, then refer to the documentation of kbase_gpu_vm_lock() about the requirements of
1732
+ * either calling during a syscall, or ensuring the allocation is small. These requirements prevent
1733
+ * an effective deadlock between the kernel's OoM killer and kbase's VMA close() handlers, which
1734
+ * could take kbase_gpu_vm_lock() too.
13661735  *
1367
- * Note : The caller must not hold vm_lock, as this could cause a deadlock if
1368
- * the kernel OoM killer runs. If the caller must allocate pages while holding
1369
- * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead.
1736
+ * If the requirements of kbase_gpu_vm_lock() cannot be satisfied when calling this function, but
1737
+ * @alloc must still be updated in a thread-safe way, then instead use
1738
+ * kbase_alloc_phy_pages_helper_locked() and restructure callers into the sequence outlined there.
13701739  *
13711740  * This function cannot be used from interrupt context
1741
+ *
1742
+ * Return: 0 if all pages have been successfully allocated. Error code otherwise
13721743  */
13731744 int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc,
13741745 		size_t nr_pages_requested);
....@@ -1378,17 +1749,19 @@
13781749  * @alloc:              allocation object to add pages to
13791750  * @pool:               Memory pool to allocate from
13801751  * @nr_pages_requested: number of physical pages to allocate
1381
- * @prealloc_sa:        Information about the partial allocation if the amount
1382
- *                      of memory requested is not a multiple of 2MB. One
1383
- *                      instance of struct kbase_sub_alloc must be allocated by
1384
- *                      the caller iff CONFIG_MALI_2MB_ALLOC is enabled.
13851752  *
1386
- * Allocates \a nr_pages_requested and updates the alloc object. This function
1387
- * does not allocate new pages from the kernel, and therefore will never trigger
1388
- * the OoM killer. Therefore, it can be run while the vm_lock is held.
1753
+ * @prealloc_sa:        Information about the partial allocation if the amount of memory requested
1754
+ *                      is not a multiple of 2MB. One instance of struct kbase_sub_alloc must be
1755
+ *                      allocated by the caller if kbdev->pagesize_2mb is enabled.
13891756  *
1390
- * As new pages can not be allocated, the caller must ensure there are
1391
- * sufficient pages in the pool. Usage of this function should look like :
1757
+ * Allocates @nr_pages_requested and updates the alloc object. This function does not allocate new
1758
+ * pages from the kernel, and therefore will never trigger the OoM killer. Therefore, it can be
1759
+ * called whilst a thread operating outside of a syscall has held the region list lock
1760
+ * (kbase_gpu_vm_lock()), as it will not cause an effective deadlock with VMA close() handlers used
1761
+ * by the OoM killer.
1762
+ *
1763
+ * As new pages can not be allocated, the caller must ensure there are sufficient pages in the
1764
+ * pool. Usage of this function should look like :
13921765  *
13931766  *   kbase_gpu_vm_lock(kctx);
13941767  *   kbase_mem_pool_lock(pool)
....@@ -1401,24 +1774,24 @@
14011774  *   }
14021775  *   kbase_alloc_phy_pages_helper_locked(pool)
14031776  *   kbase_mem_pool_unlock(pool)
1404
- *   Perform other processing that requires vm_lock...
1777
+ *   // Perform other processing that requires vm_lock...
14051778  *   kbase_gpu_vm_unlock(kctx);
14061779  *
1407
- * This ensures that the pool can be grown to the required size and that the
1408
- * allocation can complete without another thread using the newly grown pages.
1780
+ * This ensures that the pool can be grown to the required size and that the allocation can
1781
+ * complete without another thread using the newly grown pages.
14091782  *
1410
- * If CONFIG_MALI_2MB_ALLOC is defined and the allocation is >= 2MB, then
1411
- * @pool must be alloc->imported.native.kctx->lp_mem_pool. Otherwise it must be
1412
- * alloc->imported.native.kctx->mem_pool.
1413
- * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be
1414
- * pre-allocated because we must not sleep (due to the usage of kmalloc())
1415
- * whilst holding pool->pool_lock.
1416
- * @prealloc_sa shall be set to NULL if it has been consumed by this function
1417
- * to indicate that the caller must not free it.
1783
+ * If kbdev->pagesize_2mb is enabled and the allocation is >= 2MB, then @pool must be one of the
1784
+ * pools from alloc->imported.native.kctx->mem_pools.large[]. Otherwise it must be one of the
1785
+ * mempools from alloc->imported.native.kctx->mem_pools.small[].
1786
+ *
1787
+ * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be pre-allocated because we
1788
+ * must not sleep (due to the usage of kmalloc()) whilst holding pool->pool_lock.  @prealloc_sa
1789
+ * shall be set to NULL if it has been consumed by this function to indicate that the caller no
1790
+ * longer owns it and should not access it further.
1791
+ *
1792
+ * Note: Caller must hold @pool->pool_lock
14181793  *
14191794  * Return: Pointer to array of allocated pages. NULL on failure.
1420
- *
1421
- * Note : Caller must hold pool->pool_lock
14221795  */
14231796 struct tagged_addr *kbase_alloc_phy_pages_helper_locked(
14241797 		struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool,
....@@ -1428,10 +1801,10 @@
14281801 /**
14291802  * kbase_free_phy_pages_helper() - Free physical pages.
14301803  *
1431
- * Frees \a nr_pages and updates the alloc object.
1432
- *
14331804  * @alloc:            allocation object to free pages from
14341805  * @nr_pages_to_free: number of physical pages to free
1806
+ *
1807
+ * Free @nr_pages_to_free pages and updates the alloc object.
14351808  *
14361809  * Return: 0 on success, otherwise a negative error code
14371810  */
....@@ -1457,7 +1830,7 @@
14571830 		struct kbase_mem_pool *pool, struct tagged_addr *pages,
14581831 		size_t nr_pages_to_free);
14591832 
1460
-static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr)
1833
+static inline void kbase_set_dma_addr_as_priv(struct page *p, dma_addr_t dma_addr)
14611834 {
14621835 	SetPagePrivate(p);
14631836 	if (sizeof(dma_addr_t) > sizeof(p->private)) {
....@@ -1473,7 +1846,7 @@
14731846 	}
14741847 }
14751848 
1476
-static inline dma_addr_t kbase_dma_addr(struct page *p)
1849
+static inline dma_addr_t kbase_dma_addr_as_priv(struct page *p)
14771850 {
14781851 	if (sizeof(dma_addr_t) > sizeof(p->private))
14791852 		return ((dma_addr_t)page_private(p)) << PAGE_SHIFT;
....@@ -1481,9 +1854,32 @@
14811854 	return (dma_addr_t)page_private(p);
14821855 }
14831856 
1484
-static inline void kbase_clear_dma_addr(struct page *p)
1857
+static inline void kbase_clear_dma_addr_as_priv(struct page *p)
14851858 {
14861859 	ClearPagePrivate(p);
1860
+}
1861
+
1862
+static inline struct kbase_page_metadata *kbase_page_private(struct page *p)
1863
+{
1864
+	return (struct kbase_page_metadata *)page_private(p);
1865
+}
1866
+
1867
+static inline dma_addr_t kbase_dma_addr(struct page *p)
1868
+{
1869
+	if (kbase_page_migration_enabled)
1870
+		return kbase_page_private(p)->dma_addr;
1871
+
1872
+	return kbase_dma_addr_as_priv(p);
1873
+}
1874
+
1875
+static inline dma_addr_t kbase_dma_addr_from_tagged(struct tagged_addr tagged_pa)
1876
+{
1877
+	phys_addr_t pa = as_phys_addr_t(tagged_pa);
1878
+	struct page *page = pfn_to_page(PFN_DOWN(pa));
1879
+	dma_addr_t dma_addr =
1880
+		is_huge(tagged_pa) ? kbase_dma_addr_as_priv(page) : kbase_dma_addr(page);
1881
+
1882
+	return dma_addr;
14871883 }
14881884 
14891885 /**
....@@ -1529,7 +1925,7 @@
15291925  * kbase_jit_init - Initialize the JIT memory pool management
15301926  * @kctx: kbase context
15311927  *
1532
- * Returns zero on success or negative error number on failure.
1928
+ * Return: zero on success or negative error number on failure.
15331929  */
15341930 int kbase_jit_init(struct kbase_context *kctx);
15351931 
....@@ -1644,8 +2040,8 @@
16442040 		unsigned int flags);
16452041 
16462042 /**
1647
- * jit_trim_necessary_pages() - calculate and trim the least pages possible to
1648
- * satisfy a new JIT allocation
2043
+ * kbase_jit_trim_necessary_pages() - calculate and trim the least pages
2044
+ * possible to satisfy a new JIT allocation
16492045  *
16502046  * @kctx: Pointer to the kbase context
16512047  * @needed_pages: Number of JIT physical pages by which trimming is requested.
....@@ -1767,10 +2163,10 @@
17672163 /**
17682164  * kbase_has_exec_va_zone - EXEC_VA zone predicate
17692165  *
2166
+ * @kctx: kbase context
2167
+ *
17702168  * Determine whether an EXEC_VA zone has been created for the GPU address space
17712169  * of the given kbase context.
1772
- *
1773
- * @kctx: kbase context
17742170  *
17752171  * Return: True if the kbase context has an EXEC_VA zone.
17762172  */
....@@ -1779,25 +2175,38 @@
17792175 /**
17802176  * kbase_map_external_resource - Map an external resource to the GPU.
17812177  * @kctx:              kbase context.
1782
- * @reg:               The region to map.
2178
+ * @reg:               External resource to map.
17832179  * @locked_mm:         The mm_struct which has been locked for this operation.
17842180  *
1785
- * Return: The physical allocation which backs the region on success or NULL
1786
- * on failure.
2181
+ * On successful mapping, the VA region and the gpu_alloc refcounts will be
2182
+ * increased, making it safe to use and store both values directly.
2183
+ *
2184
+ * Return: Zero on success, or negative error code.
17872185  */
1788
-struct kbase_mem_phy_alloc *kbase_map_external_resource(
1789
-		struct kbase_context *kctx, struct kbase_va_region *reg,
1790
-		struct mm_struct *locked_mm);
2186
+int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_region *reg,
2187
+				struct mm_struct *locked_mm);
17912188 
17922189 /**
17932190  * kbase_unmap_external_resource - Unmap an external resource from the GPU.
17942191  * @kctx:  kbase context.
1795
- * @reg:   The region to unmap or NULL if it has already been released.
1796
- * @alloc: The physical allocation being unmapped.
2192
+ * @reg:   VA region corresponding to external resource
2193
+ *
2194
+ * On successful unmapping, the VA region and the gpu_alloc refcounts will
2195
+ * be decreased. If the refcount reaches zero, both @reg and the corresponding
2196
+ * allocation may be freed, so using them after returning from this function
2197
+ * requires the caller to explicitly check their state.
17972198  */
1798
-void kbase_unmap_external_resource(struct kbase_context *kctx,
1799
-		struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc);
2199
+void kbase_unmap_external_resource(struct kbase_context *kctx, struct kbase_va_region *reg);
18002200 
2201
+/**
2202
+ * kbase_unpin_user_buf_page - Unpin a page of a user buffer.
2203
+ * @page: page to unpin
2204
+ *
2205
+ * The caller must have ensured that there are no CPU mappings for @page (as
2206
+ * might be created from the struct kbase_mem_phy_alloc that tracks @page), and
2207
+ * that userspace will not be able to recreate the CPU mappings again.
2208
+ */
2209
+void kbase_unpin_user_buf_page(struct page *page);
18012210 
18022211 /**
18032212  * kbase_jd_user_buf_pin_pages - Pin the pages of a user buffer.
....@@ -1817,7 +2226,7 @@
18172226  * kbase_sticky_resource_init - Initialize sticky resource management.
18182227  * @kctx: kbase context
18192228  *
1820
- * Returns zero on success or negative error number on failure.
2229
+ * Return: zero on success or negative error number on failure.
18212230  */
18222231 int kbase_sticky_resource_init(struct kbase_context *kctx);
18232232 
....@@ -1879,7 +2288,7 @@
18792288 }
18802289 
18812290 /**
1882
- * kbase_mem_pool_lock - Release a memory pool
2291
+ * kbase_mem_pool_unlock - Release a memory pool
18832292  * @pool: Memory pool to lock
18842293  */
18852294 static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool)
....@@ -1939,7 +2348,7 @@
19392348  *         manage the shared interface segment of MCU firmware address space.
19402349  * @kbdev: Pointer to the kbase device
19412350  *
1942
- * Returns zero on success or negative error number on failure.
2351
+ * Return: zero on success or negative error number on failure.
19432352  */
19442353 int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev);
19452354 
....@@ -1958,7 +2367,7 @@
19582367  *
19592368  * Map a dma-buf on the GPU. The mappings are reference counted.
19602369  *
1961
- * Returns 0 on success, or a negative error code.
2370
+ * Return: 0 on success, or a negative error code.
19622371  */
19632372 int kbase_mem_umm_map(struct kbase_context *kctx,
19642373 		struct kbase_va_region *reg);
....@@ -1978,7 +2387,7 @@
19782387  * @alloc must be a valid physical allocation of type
19792388  * KBASE_MEM_TYPE_IMPORTED_UMM that was previously mapped by
19802389  * kbase_mem_umm_map(). The dma-buf attachment referenced by @alloc will
1981
- * release it's mapping reference, and if the refcount reaches 0, also be be
2390
+ * release it's mapping reference, and if the refcount reaches 0, also be
19822391  * unmapped, regardless of the value of @reg.
19832392  */
19842393 void kbase_mem_umm_unmap(struct kbase_context *kctx,
....@@ -2025,7 +2434,7 @@
20252434 		unsigned int *target_page_nr, size_t offset);
20262435 
20272436 /**
2028
- * kbase_ctx_reg_zone_end_pfn - return the end Page Frame Number of @zone
2437
+ * kbase_reg_zone_end_pfn - return the end Page Frame Number of @zone
20292438  * @zone: zone to query
20302439  *
20312440  * Return: The end of the zone corresponding to @zone
....@@ -2050,7 +2459,7 @@
20502459 	struct kbase_reg_zone *zone;
20512460 
20522461 	lockdep_assert_held(&kctx->reg_lock);
2053
-	WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits);
2462
+	WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
20542463 
20552464 	zone = &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
20562465 	*zone = (struct kbase_reg_zone){
....@@ -2073,7 +2482,7 @@
20732482 kbase_ctx_reg_zone_get_nolock(struct kbase_context *kctx,
20742483 			      unsigned long zone_bits)
20752484 {
2076
-	WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits);
2485
+	WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
20772486 
20782487 	return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
20792488 }
....@@ -2091,9 +2500,71 @@
20912500 kbase_ctx_reg_zone_get(struct kbase_context *kctx, unsigned long zone_bits)
20922501 {
20932502 	lockdep_assert_held(&kctx->reg_lock);
2094
-	WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits);
2503
+	WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
20952504 
20962505 	return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
20972506 }
20982507 
2508
+/**
2509
+ * kbase_mem_allow_alloc - Check if allocation of GPU memory is allowed
2510
+ * @kctx: Pointer to kbase context
2511
+ *
2512
+ * Don't allow the allocation of GPU memory if the ioctl has been issued
2513
+ * from the forked child process using the mali device file fd inherited from
2514
+ * the parent process.
2515
+ *
2516
+ * Return: true if allocation is allowed.
2517
+ */
2518
+static inline bool kbase_mem_allow_alloc(struct kbase_context *kctx)
2519
+{
2520
+	return (kctx->process_mm == current->mm);
2521
+}
2522
+
2523
+/**
2524
+ * kbase_mem_mmgrab - Wrapper function to take reference on mm_struct of current process
2525
+ */
2526
+static inline void kbase_mem_mmgrab(void)
2527
+{
2528
+	/* This merely takes a reference on the memory descriptor structure
2529
+	 * i.e. mm_struct of current process and not on its address space and
2530
+	 * so won't block the freeing of address space on process exit.
2531
+	 */
2532
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
2533
+	atomic_inc(&current->mm->mm_count);
2534
+#else
2535
+	mmgrab(current->mm);
2536
+#endif
2537
+}
2538
+
2539
+/**
2540
+ * kbase_mem_group_id_get - Get group ID from flags
2541
+ * @flags: Flags to pass to base_mem_alloc
2542
+ *
2543
+ * This inline function extracts the encoded group ID from flags
2544
+ * and converts it into numeric value (0~15).
2545
+ *
2546
+ * Return: group ID(0~15) extracted from the parameter
2547
+ */
2548
+static inline int kbase_mem_group_id_get(base_mem_alloc_flags flags)
2549
+{
2550
+	KBASE_DEBUG_ASSERT((flags & ~BASE_MEM_FLAGS_INPUT_MASK) == 0);
2551
+	return (int)BASE_MEM_GROUP_ID_GET(flags);
2552
+}
2553
+
2554
+/**
2555
+ * kbase_mem_group_id_set - Set group ID into base_mem_alloc_flags
2556
+ * @id: group ID(0~15) you want to encode
2557
+ *
2558
+ * This inline function encodes specific group ID into base_mem_alloc_flags.
2559
+ * Parameter 'id' should lie in-between 0 to 15.
2560
+ *
2561
+ * Return: base_mem_alloc_flags with the group ID (id) encoded
2562
+ *
2563
+ * The return value can be combined with other flags against base_mem_alloc
2564
+ * to identify a specific memory group.
2565
+ */
2566
+static inline base_mem_alloc_flags kbase_mem_group_id_set(int id)
2567
+{
2568
+	return BASE_MEM_GROUP_ID_SET(id);
2569
+}
20992570 #endif				/* _KBASE_MEM_H_ */