/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
|
/*
|
*
|
* (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
/**
|
* DOC: Base kernel memory APIs
|
*/
|
|
#ifndef _KBASE_MEM_H_
|
#define _KBASE_MEM_H_
|
|
#ifndef _KBASE_H_
|
#error "Don't include this file directly, use mali_kbase.h instead"
|
#endif
|
|
#include <linux/kref.h>
|
#include <uapi/gpu/arm/bifrost/mali_base_kernel.h>
|
#include <mali_kbase_hw.h>
|
#include "mali_kbase_pm.h"
|
#include "mali_kbase_defs.h"
|
/* Required for kbase_mem_evictable_unmake */
|
#include "mali_kbase_mem_linux.h"
|
#include "mali_kbase_mem_migrate.h"
|
#include "mali_kbase_refcount_defs.h"
|
|
static inline void kbase_process_page_usage_inc(struct kbase_context *kctx,
|
int pages);
|
|
/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */
|
|
/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by
|
* 8 pages. The MMU reads in 8 page table entries from memory at a time, if we
|
* have more than one page fault within the same 8 pages and page tables are
|
* updated accordingly, the MMU does not re-read the page table entries from
|
* memory for the subsequent page table updates and generates duplicate page
|
* faults as the page table information used by the MMU is not valid.
|
*/
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */
|
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */
|
|
/* This must always be a power of 2 */
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2)
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316)
|
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630)
|
|
/*
|
* A CPU mapping
|
*/
|
struct kbase_cpu_mapping {
|
struct list_head mappings_list;
|
struct kbase_mem_phy_alloc *alloc;
|
struct kbase_context *kctx;
|
struct kbase_va_region *region;
|
int count;
|
int free_on_close;
|
};
|
|
enum kbase_memory_type {
|
KBASE_MEM_TYPE_NATIVE,
|
KBASE_MEM_TYPE_IMPORTED_UMM,
|
KBASE_MEM_TYPE_IMPORTED_USER_BUF,
|
KBASE_MEM_TYPE_ALIAS,
|
KBASE_MEM_TYPE_RAW
|
};
|
|
/* internal structure, mirroring base_mem_aliasing_info,
|
* but with alloc instead of a gpu va (handle)
|
*/
|
struct kbase_aliased {
|
struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */
|
u64 offset; /* in pages */
|
u64 length; /* in pages */
|
};
|
|
/* Physical pages tracking object properties */
|
#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1u << 0)
|
#define KBASE_MEM_PHY_ALLOC_LARGE (1u << 1)
|
|
/* struct kbase_mem_phy_alloc - Physical pages tracking object.
|
*
|
* Set up to track N pages.
|
* N not stored here, the creator holds that info.
|
* This object only tracks how many elements are actually valid (present).
|
* Changing of nents or *pages should only happen if the kbase_mem_phy_alloc
|
* is not shared with another region or client. CPU mappings are OK to
|
* exist when changing, as long as the tracked mappings objects are
|
* updated as part of the change.
|
*
|
* @kref: number of users of this alloc
|
* @gpu_mappings: count number of times mapped on the GPU. Indicates the number
|
* of references there are to the physical pages from different
|
* GPU VA regions.
|
* @kernel_mappings: count number of times mapped on the CPU, specifically in
|
* the kernel. Indicates the number of references there are
|
* to the physical pages to prevent flag changes or shrink
|
* while maps are still held.
|
* @nents: 0..N
|
* @pages: N elements, only 0..nents are valid
|
* @mappings: List of CPU mappings of this physical memory allocation.
|
* @evict_node: Node used to store this allocation on the eviction list
|
* @evicted: Physical backing size when the pages where evicted
|
* @reg: Back reference to the region structure which created this
|
* allocation, or NULL if it has been freed.
|
* @type: type of buffer
|
* @permanent_map: Kernel side mapping of the alloc, shall never be
|
* referred directly. kbase_phy_alloc_mapping_get() &
|
* kbase_phy_alloc_mapping_put() pair should be used
|
* around access to the kernel-side CPU mapping so that
|
* mapping doesn't disappear whilst it is being accessed.
|
* @properties: Bitmask of properties, e.g. KBASE_MEM_PHY_ALLOC_LARGE.
|
* @group_id: A memory group ID to be passed to a platform-specific
|
* memory group manager, if present.
|
* Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1).
|
* @imported: member in union valid based on @a type
|
*/
|
struct kbase_mem_phy_alloc {
|
struct kref kref;
|
atomic_t gpu_mappings;
|
atomic_t kernel_mappings;
|
size_t nents;
|
struct tagged_addr *pages;
|
struct list_head mappings;
|
struct list_head evict_node;
|
size_t evicted;
|
struct kbase_va_region *reg;
|
enum kbase_memory_type type;
|
struct kbase_vmap_struct *permanent_map;
|
u8 properties;
|
u8 group_id;
|
|
union {
|
struct {
|
struct kbase_context *kctx;
|
struct dma_buf *dma_buf;
|
struct dma_buf_attachment *dma_attachment;
|
unsigned int current_mapping_usage_count;
|
struct sg_table *sgt;
|
bool need_sync;
|
} umm;
|
struct {
|
u64 stride;
|
size_t nents;
|
struct kbase_aliased *aliased;
|
} alias;
|
struct {
|
struct kbase_context *kctx;
|
/* Number of pages in this structure, including *pages.
|
* Used for kernel memory tracking.
|
*/
|
size_t nr_struct_pages;
|
} native;
|
struct kbase_alloc_import_user_buf {
|
unsigned long address;
|
unsigned long size;
|
unsigned long nr_pages;
|
struct page **pages;
|
/* top bit (1<<31) of current_mapping_usage_count
|
* specifies that this import was pinned on import
|
* See PINNED_ON_IMPORT
|
*/
|
u32 current_mapping_usage_count;
|
struct mm_struct *mm;
|
dma_addr_t *dma_addrs;
|
} user_buf;
|
} imported;
|
};
|
|
/**
|
* enum kbase_page_status - Status of a page used for page migration.
|
*
|
* @MEM_POOL: Stable state. Page is located in a memory pool and can safely
|
* be migrated.
|
* @ALLOCATE_IN_PROGRESS: Transitory state. A page is set to this status as
|
* soon as it leaves a memory pool.
|
* @SPILL_IN_PROGRESS: Transitory state. Corner case where pages in a memory
|
* pool of a dying context are being moved to the device
|
* memory pool.
|
* @NOT_MOVABLE: Stable state. Page has been allocated for an object that is
|
* not movable, but may return to be movable when the object
|
* is freed.
|
* @ALLOCATED_MAPPED: Stable state. Page has been allocated, mapped to GPU
|
* and has reference to kbase_mem_phy_alloc object.
|
* @PT_MAPPED: Stable state. Similar to ALLOCATED_MAPPED, but page doesn't
|
* reference kbase_mem_phy_alloc object. Used as a page in MMU
|
* page table.
|
* @FREE_IN_PROGRESS: Transitory state. A page is set to this status as soon as
|
* the driver manages to acquire a lock on the page while
|
* unmapping it. This status means that a memory release is
|
* happening and it's still not complete.
|
* @FREE_ISOLATED_IN_PROGRESS: Transitory state. This is a very particular corner case.
|
* A page is isolated while it is in ALLOCATED_MAPPED state,
|
* but then the driver tries to destroy the allocation.
|
* @FREE_PT_ISOLATED_IN_PROGRESS: Transitory state. This is a very particular corner case.
|
* A page is isolated while it is in PT_MAPPED state, but
|
* then the driver tries to destroy the allocation.
|
*
|
* Pages can only be migrated in stable states.
|
*/
|
enum kbase_page_status {
|
MEM_POOL = 0,
|
ALLOCATE_IN_PROGRESS,
|
SPILL_IN_PROGRESS,
|
NOT_MOVABLE,
|
ALLOCATED_MAPPED,
|
PT_MAPPED,
|
FREE_IN_PROGRESS,
|
FREE_ISOLATED_IN_PROGRESS,
|
FREE_PT_ISOLATED_IN_PROGRESS,
|
};
|
|
#define PGD_VPFN_LEVEL_MASK ((u64)0x3)
|
#define PGD_VPFN_LEVEL_GET_LEVEL(pgd_vpfn_level) (pgd_vpfn_level & PGD_VPFN_LEVEL_MASK)
|
#define PGD_VPFN_LEVEL_GET_VPFN(pgd_vpfn_level) (pgd_vpfn_level & ~PGD_VPFN_LEVEL_MASK)
|
#define PGD_VPFN_LEVEL_SET(pgd_vpfn, level) \
|
((pgd_vpfn & ~PGD_VPFN_LEVEL_MASK) | (level & PGD_VPFN_LEVEL_MASK))
|
|
/**
|
* struct kbase_page_metadata - Metadata for each page in kbase
|
*
|
* @kbdev: Pointer to kbase device.
|
* @dma_addr: DMA address mapped to page.
|
* @migrate_lock: A spinlock to protect the private metadata.
|
* @data: Member in union valid based on @status.
|
* @status: Status to keep track if page can be migrated at any
|
* given moment. MSB will indicate if page is isolated.
|
* Protected by @migrate_lock.
|
* @vmap_count: Counter of kernel mappings.
|
* @group_id: Memory group ID obtained at the time of page allocation.
|
*
|
* Each 4KB page will have a reference to this struct in the private field.
|
* This will be used to keep track of information required for Linux page
|
* migration functionality as well as address for DMA mapping.
|
*/
|
struct kbase_page_metadata {
|
dma_addr_t dma_addr;
|
spinlock_t migrate_lock;
|
|
union {
|
struct {
|
struct kbase_mem_pool *pool;
|
/* Pool could be terminated after page is isolated and therefore
|
* won't be able to get reference to kbase device.
|
*/
|
struct kbase_device *kbdev;
|
} mem_pool;
|
struct {
|
struct kbase_va_region *reg;
|
struct kbase_mmu_table *mmut;
|
u64 vpfn;
|
} mapped;
|
struct {
|
struct kbase_mmu_table *mmut;
|
u64 pgd_vpfn_level;
|
} pt_mapped;
|
struct {
|
struct kbase_device *kbdev;
|
} free_isolated;
|
struct {
|
struct kbase_device *kbdev;
|
} free_pt_isolated;
|
} data;
|
|
u8 status;
|
u8 vmap_count;
|
u8 group_id;
|
};
|
|
/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is
|
* used to signify that a buffer was pinned when it was imported. Since the
|
* reference count is limited by the number of atoms that can be submitted at
|
* once there should be no danger of overflowing into this bit.
|
* Stealing the top bit also has the benefit that
|
* current_mapping_usage_count != 0 if and only if the buffer is mapped.
|
*/
|
#define PINNED_ON_IMPORT (1<<31)
|
|
/**
|
* enum kbase_jit_report_flags - Flags for just-in-time memory allocation
|
* pressure limit functions
|
* @KBASE_JIT_REPORT_ON_ALLOC_OR_FREE: Notifying about an update happening due
|
* to a just-in-time memory allocation or free
|
*
|
* Used to control flow within pressure limit related functions, or to provide
|
* extra debugging information
|
*/
|
enum kbase_jit_report_flags {
|
KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0)
|
};
|
|
/**
|
* kbase_set_phy_alloc_page_status - Set the page migration status of the underlying
|
* physical allocation.
|
* @alloc: the physical allocation containing the pages whose metadata is going
|
* to be modified
|
* @status: the status the pages should end up in
|
*
|
* Note that this function does not go through all of the checking to ensure that
|
* proper states are set. Instead, it is only used when we change the allocation
|
* to NOT_MOVABLE or from NOT_MOVABLE to ALLOCATED_MAPPED
|
*/
|
void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc,
|
enum kbase_page_status status);
|
|
static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc)
|
{
|
KBASE_DEBUG_ASSERT(alloc);
|
/* we only track mappings of NATIVE buffers */
|
if (alloc->type == KBASE_MEM_TYPE_NATIVE)
|
atomic_inc(&alloc->gpu_mappings);
|
}
|
|
static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc)
|
{
|
KBASE_DEBUG_ASSERT(alloc);
|
/* we only track mappings of NATIVE buffers */
|
if (alloc->type == KBASE_MEM_TYPE_NATIVE)
|
if (atomic_dec_return(&alloc->gpu_mappings) < 0) {
|
pr_err("Mismatched %s:\n", __func__);
|
dump_stack();
|
}
|
}
|
|
/**
|
* kbase_mem_phy_alloc_kernel_mapped - Increment kernel_mappings counter for a
|
* memory region to prevent commit and flag
|
* changes
|
*
|
* @alloc: Pointer to physical pages tracking object
|
*/
|
static inline void
|
kbase_mem_phy_alloc_kernel_mapped(struct kbase_mem_phy_alloc *alloc)
|
{
|
atomic_inc(&alloc->kernel_mappings);
|
}
|
|
/**
|
* kbase_mem_phy_alloc_kernel_unmapped - Decrement kernel_mappings
|
* counter for a memory region to allow commit and flag changes
|
*
|
* @alloc: Pointer to physical pages tracking object
|
*/
|
static inline void
|
kbase_mem_phy_alloc_kernel_unmapped(struct kbase_mem_phy_alloc *alloc)
|
{
|
WARN_ON(atomic_dec_return(&alloc->kernel_mappings) < 0);
|
}
|
|
/**
|
* kbase_mem_is_imported - Indicate whether a memory type is imported
|
*
|
* @type: the memory type
|
*
|
* Return: true if the memory type is imported, false otherwise
|
*/
|
static inline bool kbase_mem_is_imported(enum kbase_memory_type type)
|
{
|
return (type == KBASE_MEM_TYPE_IMPORTED_UMM) ||
|
(type == KBASE_MEM_TYPE_IMPORTED_USER_BUF);
|
}
|
|
void kbase_mem_kref_free(struct kref *kref);
|
|
int kbase_mem_init(struct kbase_device *kbdev);
|
void kbase_mem_halt(struct kbase_device *kbdev);
|
void kbase_mem_term(struct kbase_device *kbdev);
|
|
static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc)
|
{
|
kref_get(&alloc->kref);
|
return alloc;
|
}
|
|
static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc)
|
{
|
kref_put(&alloc->kref, kbase_mem_kref_free);
|
return NULL;
|
}
|
|
/**
|
* struct kbase_va_region - A GPU memory region, and attributes for CPU mappings
|
*
|
* @rblink: Node in a red-black tree of memory regions within the same zone of
|
* the GPU's virtual address space.
|
* @link: Links to neighboring items in a list of growable memory regions
|
* that triggered incremental rendering by growing too much.
|
* @rbtree: Backlink to the red-black tree of memory regions.
|
* @start_pfn: The Page Frame Number in GPU virtual address space.
|
* @user_data: The address of GPU command queue when VA region represents
|
* a ring buffer.
|
* @nr_pages: The size of the region in pages.
|
* @initial_commit: Initial commit, for aligning the start address and
|
* correctly growing KBASE_REG_TILER_ALIGN_TOP regions.
|
* @threshold_pages: If non-zero and the amount of memory committed to a region
|
* that can grow on page fault exceeds this number of pages
|
* then the driver switches to incremental rendering.
|
* @flags: Flags
|
* @extension: Number of pages allocated on page fault.
|
* @cpu_alloc: The physical memory we mmap to the CPU when mapping this region.
|
* @gpu_alloc: The physical memory we mmap to the GPU when mapping this region.
|
* @jit_node: Links to neighboring regions in the just-in-time memory pool.
|
* @jit_usage_id: The last just-in-time memory usage ID for this region.
|
* @jit_bin_id: The just-in-time memory bin this region came from.
|
* @va_refcnt: Number of users of this region. Protected by reg_lock.
|
* @no_user_free_count: Number of contexts that want to prevent the region
|
* from being freed by userspace.
|
* @heap_info_gpu_addr: Pointer to an object in GPU memory defining an end of
|
* an allocated region
|
* The object can be one of:
|
* - u32 value defining the size of the region
|
* - u64 pointer first unused byte in the region
|
* The interpretation of the object depends on
|
* BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE flag in
|
* jit_info_flags - if it is set, the heap info object
|
* should be interpreted as size.
|
* @used_pages: The current estimate of the number of pages used, which in
|
* normal use is either:
|
* - the initial estimate == va_pages
|
* - the actual pages used, as found by a JIT usage report
|
* Note that since the value is calculated from GPU memory after a
|
* JIT usage report, at any point in time it is allowed to take a
|
* random value that is no greater than va_pages (e.g. it may be
|
* greater than gpu_alloc->nents)
|
*/
|
struct kbase_va_region {
|
struct rb_node rblink;
|
struct list_head link;
|
struct rb_root *rbtree;
|
u64 start_pfn;
|
void *user_data;
|
size_t nr_pages;
|
size_t initial_commit;
|
size_t threshold_pages;
|
|
/* Free region */
|
#define KBASE_REG_FREE (1ul << 0)
|
/* CPU write access */
|
#define KBASE_REG_CPU_WR (1ul << 1)
|
/* GPU write access */
|
#define KBASE_REG_GPU_WR (1ul << 2)
|
/* No eXecute flag */
|
#define KBASE_REG_GPU_NX (1ul << 3)
|
/* Is CPU cached? */
|
#define KBASE_REG_CPU_CACHED (1ul << 4)
|
/* Is GPU cached?
|
* Some components within the GPU might only be able to access memory that is
|
* GPU cacheable. Refer to the specific GPU implementation for more details.
|
*/
|
#define KBASE_REG_GPU_CACHED (1ul << 5)
|
|
#define KBASE_REG_GROWABLE (1ul << 6)
|
/* Can grow on pf? */
|
#define KBASE_REG_PF_GROW (1ul << 7)
|
|
/* Allocation doesn't straddle the 4GB boundary in GPU virtual space */
|
#define KBASE_REG_GPU_VA_SAME_4GB_PAGE (1ul << 8)
|
|
/* inner shareable coherency */
|
#define KBASE_REG_SHARE_IN (1ul << 9)
|
/* inner & outer shareable coherency */
|
#define KBASE_REG_SHARE_BOTH (1ul << 10)
|
|
#if MALI_USE_CSF
|
/* Space for 8 different zones */
|
#define KBASE_REG_ZONE_BITS 3
|
#else
|
/* Space for 4 different zones */
|
#define KBASE_REG_ZONE_BITS 2
|
#endif
|
|
#define KBASE_REG_ZONE_MASK (((1 << KBASE_REG_ZONE_BITS) - 1ul) << 11)
|
#define KBASE_REG_ZONE(x) (((x) & ((1 << KBASE_REG_ZONE_BITS) - 1ul)) << 11)
|
#define KBASE_REG_ZONE_IDX(x) (((x) & KBASE_REG_ZONE_MASK) >> 11)
|
|
#if KBASE_REG_ZONE_MAX > (1 << KBASE_REG_ZONE_BITS)
|
#error "Too many zones for the number of zone bits defined"
|
#endif
|
|
/* GPU read access */
|
#define KBASE_REG_GPU_RD (1ul << 14)
|
/* CPU read access */
|
#define KBASE_REG_CPU_RD (1ul << 15)
|
|
/* Index of chosen MEMATTR for this region (0..7) */
|
#define KBASE_REG_MEMATTR_MASK (7ul << 16)
|
#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16)
|
#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16)
|
|
#define KBASE_REG_PROTECTED (1ul << 19)
|
|
/* Region belongs to a shrinker.
|
*
|
* This can either mean that it is part of the JIT/Ephemeral or tiler heap
|
* shrinker paths. Should be removed only after making sure that there are
|
* no references remaining to it in these paths, as it may cause the physical
|
* backing of the region to disappear during use.
|
*/
|
#define KBASE_REG_DONT_NEED (1ul << 20)
|
|
/* Imported buffer is padded? */
|
#define KBASE_REG_IMPORT_PAD (1ul << 21)
|
|
#if MALI_USE_CSF
|
/* CSF event memory */
|
#define KBASE_REG_CSF_EVENT (1ul << 22)
|
#else
|
/* Bit 22 is reserved.
|
*
|
* Do not remove, use the next unreserved bit for new flags
|
*/
|
#define KBASE_REG_RESERVED_BIT_22 (1ul << 22)
|
#endif
|
|
#if !MALI_USE_CSF
|
/* The top of the initial commit is aligned to extension pages.
|
* Extent must be a power of 2
|
*/
|
#define KBASE_REG_TILER_ALIGN_TOP (1ul << 23)
|
#else
|
/* Bit 23 is reserved.
|
*
|
* Do not remove, use the next unreserved bit for new flags
|
*/
|
#define KBASE_REG_RESERVED_BIT_23 (1ul << 23)
|
#endif /* !MALI_USE_CSF */
|
|
/* Bit 24 is currently unused and is available for use for a new flag */
|
|
/* Memory has permanent kernel side mapping */
|
#define KBASE_REG_PERMANENT_KERNEL_MAPPING (1ul << 25)
|
|
/* GPU VA region has been freed by the userspace, but still remains allocated
|
* due to the reference held by CPU mappings created on the GPU VA region.
|
*
|
* A region with this flag set has had kbase_gpu_munmap() called on it, but can
|
* still be looked-up in the region tracker as a non-free region. Hence must
|
* not create or update any more GPU mappings on such regions because they will
|
* not be unmapped when the region is finally destroyed.
|
*
|
* Since such regions are still present in the region tracker, new allocations
|
* attempted with BASE_MEM_SAME_VA might fail if their address intersects with
|
* a region with this flag set.
|
*
|
* In addition, this flag indicates the gpu_alloc member might no longer valid
|
* e.g. in infinite cache simulation.
|
*/
|
#define KBASE_REG_VA_FREED (1ul << 26)
|
|
/* If set, the heap info address points to a u32 holding the used size in bytes;
|
* otherwise it points to a u64 holding the lowest address of unused memory.
|
*/
|
#define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27)
|
|
/* Allocation is actively used for JIT memory */
|
#define KBASE_REG_ACTIVE_JIT_ALLOC (1ul << 28)
|
|
#if MALI_USE_CSF
|
/* This flag only applies to allocations in the EXEC_FIXED_VA and FIXED_VA
|
* memory zones, and it determines whether they were created with a fixed
|
* GPU VA address requested by the user.
|
*/
|
#define KBASE_REG_FIXED_ADDRESS (1ul << 29)
|
#else
|
#define KBASE_REG_RESERVED_BIT_29 (1ul << 29)
|
#endif
|
|
#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0)
|
|
#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(1)
|
#define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT)
|
|
#if MALI_USE_CSF
|
/* only used with 32-bit clients */
|
/* On a 32bit platform, custom VA should be wired from 4GB to 2^(43).
|
*/
|
#define KBASE_REG_ZONE_CUSTOM_VA_SIZE \
|
(((1ULL << 43) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
|
#else
|
/* only used with 32-bit clients */
|
/* On a 32bit platform, custom VA should be wired from 4GB to the VA limit of the
|
* GPU. Unfortunately, the Linux mmap() interface limits us to 2^32 pages (2^44
|
* bytes, see mmap64 man page for reference). So we put the default limit to the
|
* maximum possible on Linux and shrink it down, if required by the GPU, during
|
* initialization.
|
*/
|
#define KBASE_REG_ZONE_CUSTOM_VA_SIZE \
|
(((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
|
/* end 32-bit clients only */
|
#endif
|
|
/* The starting address and size of the GPU-executable zone are dynamic
|
* and depend on the platform and the number of pages requested by the
|
* user process, with an upper limit of 4 GB.
|
*/
|
#define KBASE_REG_ZONE_EXEC_VA KBASE_REG_ZONE(2)
|
#define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */
|
|
#if MALI_USE_CSF
|
#define KBASE_REG_ZONE_MCU_SHARED KBASE_REG_ZONE(3)
|
#define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT)
|
#define KBASE_REG_ZONE_MCU_SHARED_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - \
|
KBASE_REG_ZONE_MCU_SHARED_BASE)
|
|
/* For CSF GPUs, the EXEC_VA zone is always 4GB in size, and starts at 2^47 for 64-bit
|
* clients, and 2^43 for 32-bit clients.
|
*/
|
#define KBASE_REG_ZONE_EXEC_VA_BASE_64 ((1ULL << 47) >> PAGE_SHIFT)
|
#define KBASE_REG_ZONE_EXEC_VA_BASE_32 ((1ULL << 43) >> PAGE_SHIFT)
|
#define KBASE_REG_ZONE_EXEC_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES
|
|
/* Executable zone supporting FIXED/FIXABLE allocations.
|
* It is always 4GB in size.
|
*/
|
|
#define KBASE_REG_ZONE_EXEC_FIXED_VA KBASE_REG_ZONE(4)
|
#define KBASE_REG_ZONE_EXEC_FIXED_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES
|
|
/* Non-executable zone supporting FIXED/FIXABLE allocations.
|
* It extends from (2^47) up to (2^48)-1, for 64-bit userspace clients, and from
|
* (2^43) up to (2^44)-1 for 32-bit userspace clients.
|
*/
|
#define KBASE_REG_ZONE_FIXED_VA KBASE_REG_ZONE(5)
|
|
/* Again - 32-bit userspace cannot map addresses beyond 2^44, but 64-bit can - and so
|
* the end of the FIXED_VA zone for 64-bit clients is (2^48)-1.
|
*/
|
#define KBASE_REG_ZONE_FIXED_VA_END_64 ((1ULL << 48) >> PAGE_SHIFT)
|
#define KBASE_REG_ZONE_FIXED_VA_END_32 ((1ULL << 44) >> PAGE_SHIFT)
|
|
#endif
|
|
unsigned long flags;
|
size_t extension;
|
struct kbase_mem_phy_alloc *cpu_alloc;
|
struct kbase_mem_phy_alloc *gpu_alloc;
|
struct list_head jit_node;
|
u16 jit_usage_id;
|
u8 jit_bin_id;
|
|
#if MALI_JIT_PRESSURE_LIMIT_BASE
|
/* Pointer to an object in GPU memory defining an end of an allocated
|
* region
|
*
|
* The object can be one of:
|
* - u32 value defining the size of the region
|
* - u64 pointer first unused byte in the region
|
*
|
* The interpretation of the object depends on
|
* BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE flag in jit_info_flags - if it is
|
* set, the heap info object should be interpreted as size.
|
*/
|
u64 heap_info_gpu_addr;
|
|
/* The current estimate of the number of pages used, which in normal
|
* use is either:
|
* - the initial estimate == va_pages
|
* - the actual pages used, as found by a JIT usage report
|
*
|
* Note that since the value is calculated from GPU memory after a JIT
|
* usage report, at any point in time it is allowed to take a random
|
* value that is no greater than va_pages (e.g. it may be greater than
|
* gpu_alloc->nents)
|
*/
|
size_t used_pages;
|
#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
|
|
kbase_refcount_t va_refcnt;
|
atomic_t no_user_free_count;
|
};
|
|
/**
|
* kbase_is_ctx_reg_zone - determine whether a KBASE_REG_ZONE_<...> is for a
|
* context or for a device
|
* @zone_bits: A KBASE_REG_ZONE_<...> to query
|
*
|
* Return: True if the zone for @zone_bits is a context zone, False otherwise
|
*/
|
static inline bool kbase_is_ctx_reg_zone(unsigned long zone_bits)
|
{
|
WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits);
|
return (zone_bits == KBASE_REG_ZONE_SAME_VA ||
|
#if MALI_USE_CSF
|
zone_bits == KBASE_REG_ZONE_EXEC_FIXED_VA || zone_bits == KBASE_REG_ZONE_FIXED_VA ||
|
#endif
|
zone_bits == KBASE_REG_ZONE_CUSTOM_VA || zone_bits == KBASE_REG_ZONE_EXEC_VA);
|
}
|
|
/* Special marker for failed JIT allocations that still must be marked as
|
* in-use
|
*/
|
#define KBASE_RESERVED_REG_JIT_ALLOC ((struct kbase_va_region *)-1)
|
|
static inline bool kbase_is_region_free(struct kbase_va_region *reg)
|
{
|
return (!reg || reg->flags & KBASE_REG_FREE);
|
}
|
|
static inline bool kbase_is_region_invalid(struct kbase_va_region *reg)
|
{
|
return (!reg || reg->flags & KBASE_REG_VA_FREED);
|
}
|
|
static inline bool kbase_is_region_invalid_or_free(struct kbase_va_region *reg)
|
{
|
/* Possibly not all functions that find regions would be using this
|
* helper, so they need to be checked when maintaining this function.
|
*/
|
return (kbase_is_region_invalid(reg) || kbase_is_region_free(reg));
|
}
|
|
/**
|
* kbase_is_region_shrinkable - Check if a region is "shrinkable".
|
* A shrinkable regions is a region for which its backing pages (reg->gpu_alloc->pages)
|
* can be freed at any point, even though the kbase_va_region structure itself
|
* may have been refcounted.
|
* Regions that aren't on a shrinker, but could be shrunk at any point in future
|
* without warning are still considered "shrinkable" (e.g. Active JIT allocs)
|
*
|
* @reg: Pointer to region
|
*
|
* Return: true if the region is "shrinkable", false if not.
|
*/
|
static inline bool kbase_is_region_shrinkable(struct kbase_va_region *reg)
|
{
|
return (reg->flags & KBASE_REG_DONT_NEED) || (reg->flags & KBASE_REG_ACTIVE_JIT_ALLOC);
|
}
|
|
void kbase_remove_va_region(struct kbase_device *kbdev,
|
struct kbase_va_region *reg);
|
static inline void kbase_region_refcnt_free(struct kbase_device *kbdev,
|
struct kbase_va_region *reg)
|
{
|
/* If region was mapped then remove va region*/
|
if (reg->start_pfn)
|
kbase_remove_va_region(kbdev, reg);
|
|
/* To detect use-after-free in debug builds */
|
KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE);
|
kfree(reg);
|
}
|
|
static inline struct kbase_va_region *kbase_va_region_alloc_get(
|
struct kbase_context *kctx, struct kbase_va_region *region)
|
{
|
WARN_ON(!kbase_refcount_read(®ion->va_refcnt));
|
WARN_ON(kbase_refcount_read(®ion->va_refcnt) == INT_MAX);
|
|
dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %pK\n",
|
kbase_refcount_read(®ion->va_refcnt), (void *)region);
|
kbase_refcount_inc(®ion->va_refcnt);
|
|
return region;
|
}
|
|
static inline struct kbase_va_region *kbase_va_region_alloc_put(
|
struct kbase_context *kctx, struct kbase_va_region *region)
|
{
|
WARN_ON(kbase_refcount_read(®ion->va_refcnt) <= 0);
|
WARN_ON(region->flags & KBASE_REG_FREE);
|
|
if (kbase_refcount_dec_and_test(®ion->va_refcnt))
|
kbase_region_refcnt_free(kctx->kbdev, region);
|
else
|
dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %pK\n",
|
kbase_refcount_read(®ion->va_refcnt), (void *)region);
|
|
return NULL;
|
}
|
|
/**
|
* kbase_va_region_is_no_user_free - Check if user free is forbidden for the region.
|
* A region that must not be freed by userspace indicates that it is owned by some other
|
* kbase subsystem, for example tiler heaps, JIT memory or CSF queues.
|
* Such regions must not be shrunk (i.e. have their backing pages freed), except by the
|
* current owner.
|
* Hence, callers cannot rely on this check alone to determine if a region might be shrunk
|
* by any part of kbase. Instead they should use kbase_is_region_shrinkable().
|
*
|
* @region: Pointer to region.
|
*
|
* Return: true if userspace cannot free the region, false if userspace can free the region.
|
*/
|
static inline bool kbase_va_region_is_no_user_free(struct kbase_va_region *region)
|
{
|
return atomic_read(®ion->no_user_free_count) > 0;
|
}
|
|
/**
|
* kbase_va_region_no_user_free_inc - Increment "no user free" count for a region.
|
* Calling this function will prevent the region to be shrunk by parts of kbase that
|
* don't own the region (as long as the count stays above zero). Refer to
|
* kbase_va_region_is_no_user_free() for more information.
|
*
|
* @region: Pointer to region (not shrinkable).
|
*
|
* Return: the pointer to the region passed as argument.
|
*/
|
static inline void kbase_va_region_no_user_free_inc(struct kbase_va_region *region)
|
{
|
WARN_ON(kbase_is_region_shrinkable(region));
|
WARN_ON(atomic_read(®ion->no_user_free_count) == INT_MAX);
|
|
/* non-atomic as kctx->reg_lock is held */
|
atomic_inc(®ion->no_user_free_count);
|
}
|
|
/**
|
* kbase_va_region_no_user_free_dec - Decrement "no user free" count for a region.
|
*
|
* @region: Pointer to region (not shrinkable).
|
*/
|
static inline void kbase_va_region_no_user_free_dec(struct kbase_va_region *region)
|
{
|
WARN_ON(!kbase_va_region_is_no_user_free(region));
|
|
atomic_dec(®ion->no_user_free_count);
|
}
|
|
/* Common functions */
|
static inline struct tagged_addr *kbase_get_cpu_phy_pages(
|
struct kbase_va_region *reg)
|
{
|
KBASE_DEBUG_ASSERT(reg);
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->gpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
|
|
return reg->cpu_alloc->pages;
|
}
|
|
static inline struct tagged_addr *kbase_get_gpu_phy_pages(
|
struct kbase_va_region *reg)
|
{
|
KBASE_DEBUG_ASSERT(reg);
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->gpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
|
|
return reg->gpu_alloc->pages;
|
}
|
|
static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg)
|
{
|
KBASE_DEBUG_ASSERT(reg);
|
/* if no alloc object the backed size naturally is 0 */
|
if (!reg->cpu_alloc)
|
return 0;
|
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->gpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);
|
|
return reg->cpu_alloc->nents;
|
}
|
|
#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */
|
|
static inline struct kbase_mem_phy_alloc *kbase_alloc_create(
|
struct kbase_context *kctx, size_t nr_pages,
|
enum kbase_memory_type type, int group_id)
|
{
|
struct kbase_mem_phy_alloc *alloc;
|
size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages;
|
size_t per_page_size = sizeof(*alloc->pages);
|
|
/* Imported pages may have page private data already in use */
|
if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) {
|
alloc_size += nr_pages *
|
sizeof(*alloc->imported.user_buf.dma_addrs);
|
per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs);
|
}
|
|
/*
|
* Prevent nr_pages*per_page_size + sizeof(*alloc) from
|
* wrapping around.
|
*/
|
if (nr_pages > ((((size_t) -1) - sizeof(*alloc))
|
/ per_page_size))
|
return ERR_PTR(-ENOMEM);
|
|
/* Allocate based on the size to reduce internal fragmentation of vmem */
|
if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD)
|
alloc = vzalloc(alloc_size);
|
else
|
alloc = kzalloc(alloc_size, GFP_KERNEL);
|
|
if (!alloc)
|
return ERR_PTR(-ENOMEM);
|
|
if (type == KBASE_MEM_TYPE_NATIVE) {
|
alloc->imported.native.nr_struct_pages =
|
(alloc_size + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
|
kbase_process_page_usage_inc(kctx,
|
alloc->imported.native.nr_struct_pages);
|
}
|
|
/* Store allocation method */
|
if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD)
|
alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE;
|
|
kref_init(&alloc->kref);
|
atomic_set(&alloc->gpu_mappings, 0);
|
atomic_set(&alloc->kernel_mappings, 0);
|
alloc->nents = 0;
|
alloc->pages = (void *)(alloc + 1);
|
INIT_LIST_HEAD(&alloc->mappings);
|
alloc->type = type;
|
alloc->group_id = group_id;
|
|
if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF)
|
alloc->imported.user_buf.dma_addrs =
|
(void *) (alloc->pages + nr_pages);
|
|
return alloc;
|
}
|
|
static inline int kbase_reg_prepare_native(struct kbase_va_region *reg,
|
struct kbase_context *kctx, int group_id)
|
{
|
KBASE_DEBUG_ASSERT(reg);
|
KBASE_DEBUG_ASSERT(!reg->cpu_alloc);
|
KBASE_DEBUG_ASSERT(!reg->gpu_alloc);
|
KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE);
|
|
reg->cpu_alloc = kbase_alloc_create(kctx, reg->nr_pages,
|
KBASE_MEM_TYPE_NATIVE, group_id);
|
if (IS_ERR(reg->cpu_alloc))
|
return PTR_ERR(reg->cpu_alloc);
|
else if (!reg->cpu_alloc)
|
return -ENOMEM;
|
|
reg->cpu_alloc->imported.native.kctx = kctx;
|
if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE)
|
&& (reg->flags & KBASE_REG_CPU_CACHED)) {
|
reg->gpu_alloc = kbase_alloc_create(kctx, reg->nr_pages,
|
KBASE_MEM_TYPE_NATIVE, group_id);
|
if (IS_ERR_OR_NULL(reg->gpu_alloc)) {
|
kbase_mem_phy_alloc_put(reg->cpu_alloc);
|
return -ENOMEM;
|
}
|
reg->gpu_alloc->imported.native.kctx = kctx;
|
} else {
|
reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
|
}
|
|
mutex_lock(&kctx->jit_evict_lock);
|
INIT_LIST_HEAD(®->cpu_alloc->evict_node);
|
INIT_LIST_HEAD(®->gpu_alloc->evict_node);
|
mutex_unlock(&kctx->jit_evict_lock);
|
|
reg->flags &= ~KBASE_REG_FREE;
|
|
return 0;
|
}
|
|
/*
|
* Max size for kbdev memory pool (in pages)
|
*/
|
#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT)
|
|
/*
|
* Max size for kctx memory pool (in pages)
|
*/
|
#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT)
|
|
/*
|
* The order required for a 2MB page allocation (2^order * 4KB = 2MB)
|
*/
|
#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9
|
|
/*
|
* The order required for a 4KB page allocation
|
*/
|
#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0
|
|
/**
|
* kbase_mem_pool_config_set_max_size - Set maximum number of free pages in
|
* initial configuration of a memory pool
|
*
|
* @config: Initial configuration for a physical memory pool
|
* @max_size: Maximum number of free pages that a pool created from
|
* @config can hold
|
*/
|
static inline void kbase_mem_pool_config_set_max_size(
|
struct kbase_mem_pool_config *const config, size_t const max_size)
|
{
|
WRITE_ONCE(config->max_size, max_size);
|
}
|
|
/**
|
* kbase_mem_pool_config_get_max_size - Get maximum number of free pages from
|
* initial configuration of a memory pool
|
*
|
* @config: Initial configuration for a physical memory pool
|
*
|
* Return: Maximum number of free pages that a pool created from @config
|
* can hold
|
*/
|
static inline size_t kbase_mem_pool_config_get_max_size(
|
const struct kbase_mem_pool_config *const config)
|
{
|
return READ_ONCE(config->max_size);
|
}
|
|
/**
|
* kbase_mem_pool_init - Create a memory pool for a kbase device
|
* @pool: Memory pool to initialize
|
* @config: Initial configuration for the memory pool
|
* @order: Page order for physical page size (order=0=>4kB, order=9=>2MB)
|
* @group_id: A memory group ID to be passed to a platform-specific
|
* memory group manager, if present.
|
* Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1).
|
* @kbdev: Kbase device where memory is used
|
* @next_pool: Pointer to the next pool or NULL.
|
*
|
* Allocations from @pool are in whole pages. Each @pool has a free list where
|
* pages can be quickly allocated from. The free list is initially empty and
|
* filled whenever pages are freed back to the pool. The number of free pages
|
* in the pool will in general not exceed @max_size, but the pool may in
|
* certain corner cases grow above @max_size.
|
*
|
* If @next_pool is not NULL, we will allocate from @next_pool before going to
|
* the memory group manager. Similarly pages can spill over to @next_pool when
|
* @pool is full. Pages are zeroed before they spill over to another pool, to
|
* prevent leaking information between applications.
|
*
|
* A shrinker is registered so that Linux mm can reclaim pages from the pool as
|
* needed.
|
*
|
* Return: 0 on success, negative -errno on error
|
*/
|
int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config,
|
unsigned int order, int group_id, struct kbase_device *kbdev,
|
struct kbase_mem_pool *next_pool);
|
|
/**
|
* kbase_mem_pool_term - Destroy a memory pool
|
* @pool: Memory pool to destroy
|
*
|
* Pages in the pool will spill over to @next_pool (if available) or freed to
|
* the kernel.
|
*/
|
void kbase_mem_pool_term(struct kbase_mem_pool *pool);
|
|
/**
|
* kbase_mem_pool_alloc - Allocate a page from memory pool
|
* @pool: Memory pool to allocate from
|
*
|
* Allocations from the pool are made as follows:
|
* 1. If there are free pages in the pool, allocate a page from @pool.
|
* 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page
|
* from @next_pool.
|
* 3. Return NULL if no memory in the pool
|
*
|
* Return: Pointer to allocated page, or NULL if allocation failed.
|
*
|
* Note : This function should not be used if the pool lock is held. Use
|
* kbase_mem_pool_alloc_locked() instead.
|
*/
|
struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool);
|
|
/**
|
* kbase_mem_pool_alloc_locked - Allocate a page from memory pool
|
* @pool: Memory pool to allocate from
|
*
|
* If there are free pages in the pool, this function allocates a page from
|
* @pool. This function does not use @next_pool.
|
*
|
* Return: Pointer to allocated page, or NULL if allocation failed.
|
*
|
* Note : Caller must hold the pool lock.
|
*/
|
struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool);
|
|
/**
|
* kbase_mem_pool_free - Free a page to memory pool
|
* @pool: Memory pool where page should be freed
|
* @page: Page to free to the pool
|
* @dirty: Whether some of the page may be dirty in the cache.
|
*
|
* Pages are freed to the pool as follows:
|
* 1. If @pool is not full, add @page to @pool.
|
* 2. Otherwise, if @next_pool is not NULL and not full, add @page to
|
* @next_pool.
|
* 3. Finally, free @page to the kernel.
|
*
|
* Note : This function should not be used if the pool lock is held. Use
|
* kbase_mem_pool_free_locked() instead.
|
*/
|
void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page,
|
bool dirty);
|
|
/**
|
* kbase_mem_pool_free_locked - Free a page to memory pool
|
* @pool: Memory pool where page should be freed
|
* @p: Page to free to the pool
|
* @dirty: Whether some of the page may be dirty in the cache.
|
*
|
* If @pool is not full, this function adds @page to @pool. Otherwise, @page is
|
* freed to the kernel. This function does not use @next_pool.
|
*
|
* Note : Caller must hold the pool lock.
|
*/
|
void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p,
|
bool dirty);
|
|
/**
|
* kbase_mem_pool_alloc_pages - Allocate pages from memory pool
|
* @pool: Memory pool to allocate from
|
* @nr_4k_pages: Number of pages to allocate
|
* @pages: Pointer to array where the physical address of the allocated
|
* pages will be stored.
|
* @partial_allowed: If fewer pages allocated is allowed
|
* @page_owner: Pointer to the task that created the Kbase context for which
|
* the pages are being allocated. It can be NULL if the pages
|
* won't be associated with any Kbase context.
|
*
|
* Like kbase_mem_pool_alloc() but optimized for allocating many pages.
|
*
|
* Return:
|
* On success number of pages allocated (could be less than nr_pages if
|
* partial_allowed).
|
* On error an error code.
|
*
|
* Note : This function should not be used if the pool lock is held. Use
|
* kbase_mem_pool_alloc_pages_locked() instead.
|
*
|
* The caller must not hold vm_lock, as this could cause a deadlock if
|
* the kernel OoM killer runs. If the caller must allocate pages while holding
|
* this lock, it should use kbase_mem_pool_alloc_pages_locked() instead.
|
*/
|
int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages,
|
struct tagged_addr *pages, bool partial_allowed,
|
struct task_struct *page_owner);
|
|
/**
|
* kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool
|
* @pool: Memory pool to allocate from
|
* @nr_4k_pages: Number of pages to allocate
|
* @pages: Pointer to array where the physical address of the allocated
|
* pages will be stored.
|
*
|
* Like kbase_mem_pool_alloc() but optimized for allocating many pages. This
|
* version does not allocate new pages from the kernel, and therefore will never
|
* trigger the OoM killer. Therefore, it can be run while the vm_lock is held.
|
*
|
* As new pages can not be allocated, the caller must ensure there are
|
* sufficient pages in the pool. Usage of this function should look like :
|
*
|
* kbase_gpu_vm_lock(kctx);
|
* kbase_mem_pool_lock(pool)
|
* while (kbase_mem_pool_size(pool) < pages_required) {
|
* kbase_mem_pool_unlock(pool)
|
* kbase_gpu_vm_unlock(kctx);
|
* kbase_mem_pool_grow(pool)
|
* kbase_gpu_vm_lock(kctx);
|
* kbase_mem_pool_lock(pool)
|
* }
|
* kbase_mem_pool_alloc_pages_locked(pool)
|
* kbase_mem_pool_unlock(pool)
|
* Perform other processing that requires vm_lock...
|
* kbase_gpu_vm_unlock(kctx);
|
*
|
* This ensures that the pool can be grown to the required size and that the
|
* allocation can complete without another thread using the newly grown pages.
|
*
|
* Return:
|
* On success number of pages allocated.
|
* On error an error code.
|
*
|
* Note : Caller must hold the pool lock.
|
*/
|
int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool,
|
size_t nr_4k_pages, struct tagged_addr *pages);
|
|
/**
|
* kbase_mem_pool_free_pages - Free pages to memory pool
|
* @pool: Memory pool where pages should be freed
|
* @nr_pages: Number of pages to free
|
* @pages: Pointer to array holding the physical addresses of the pages to
|
* free.
|
* @dirty: Whether any pages may be dirty in the cache.
|
* @reclaimed: Whether the pages where reclaimable and thus should bypass
|
* the pool and go straight to the kernel.
|
*
|
* Like kbase_mem_pool_free() but optimized for freeing many pages.
|
*/
|
void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages,
|
struct tagged_addr *pages, bool dirty, bool reclaimed);
|
|
/**
|
* kbase_mem_pool_free_pages_locked - Free pages to memory pool
|
* @pool: Memory pool where pages should be freed
|
* @nr_pages: Number of pages to free
|
* @pages: Pointer to array holding the physical addresses of the pages to
|
* free.
|
* @dirty: Whether any pages may be dirty in the cache.
|
* @reclaimed: Whether the pages where reclaimable and thus should bypass
|
* the pool and go straight to the kernel.
|
*
|
* Like kbase_mem_pool_free() but optimized for freeing many pages.
|
*/
|
void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool,
|
size_t nr_pages, struct tagged_addr *pages, bool dirty,
|
bool reclaimed);
|
|
/**
|
* kbase_mem_pool_size - Get number of free pages in memory pool
|
* @pool: Memory pool to inspect
|
*
|
* Note: the size of the pool may in certain corner cases exceed @max_size!
|
*
|
* Return: Number of free pages in the pool
|
*/
|
static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool)
|
{
|
return READ_ONCE(pool->cur_size);
|
}
|
|
/**
|
* kbase_mem_pool_max_size - Get maximum number of free pages in memory pool
|
* @pool: Memory pool to inspect
|
*
|
* Return: Maximum number of free pages in the pool
|
*/
|
static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool)
|
{
|
return pool->max_size;
|
}
|
|
|
/**
|
* kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool
|
* @pool: Memory pool to inspect
|
* @max_size: Maximum number of free pages the pool can hold
|
*
|
* If @max_size is reduced, the pool will be shrunk to adhere to the new limit.
|
* For details see kbase_mem_pool_shrink().
|
*/
|
void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size);
|
|
/**
|
* kbase_mem_pool_grow - Grow the pool
|
* @pool: Memory pool to grow
|
* @nr_to_grow: Number of pages to add to the pool
|
* @page_owner: Pointer to the task that created the Kbase context for which
|
* the memory pool is being grown. It can be NULL if the pages
|
* to be allocated won't be associated with any Kbase context.
|
*
|
* Adds @nr_to_grow pages to the pool. Note that this may cause the pool to
|
* become larger than the maximum size specified.
|
*
|
* Return: 0 on success, -ENOMEM if unable to allocate sufficent pages
|
*/
|
int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow,
|
struct task_struct *page_owner);
|
|
/**
|
* kbase_mem_pool_trim - Grow or shrink the pool to a new size
|
* @pool: Memory pool to trim
|
* @new_size: New number of pages in the pool
|
*
|
* If @new_size > @cur_size, fill the pool with new pages from the kernel, but
|
* not above the max_size for the pool.
|
* If @new_size < @cur_size, shrink the pool by freeing pages to the kernel.
|
*/
|
void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size);
|
|
/**
|
* kbase_mem_pool_mark_dying - Mark that this pool is dying
|
* @pool: Memory pool
|
*
|
* This will cause any ongoing allocation operations (eg growing on page fault)
|
* to be terminated.
|
*/
|
void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool);
|
|
/**
|
* kbase_mem_alloc_page - Allocate a new page for a device
|
* @pool: Memory pool to allocate a page from
|
*
|
* Most uses should use kbase_mem_pool_alloc to allocate a page. However that
|
* function can fail in the event the pool is empty.
|
*
|
* Return: A new page or NULL if no memory
|
*/
|
struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool);
|
|
/**
|
* kbase_mem_pool_free_page - Free a page from a memory pool.
|
* @pool: Memory pool to free a page from
|
* @p: Page to free
|
*
|
* This will free any associated data stored for the page and release
|
* the page back to the kernel.
|
*/
|
void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p);
|
|
/**
|
* kbase_region_tracker_init - Initialize the region tracker data structure
|
* @kctx: kbase context
|
*
|
* Return: 0 if success, negative error code otherwise.
|
*/
|
int kbase_region_tracker_init(struct kbase_context *kctx);
|
|
/**
|
* kbase_region_tracker_init_jit - Initialize the just-in-time memory
|
* allocation region
|
* @kctx: Kbase context.
|
* @jit_va_pages: Size of the JIT region in pages.
|
* @max_allocations: Maximum number of allocations allowed for the JIT region.
|
* Valid range is 0..%BASE_JIT_ALLOC_COUNT.
|
* @trim_level: Trim level for the JIT region.
|
* Valid range is 0..%BASE_JIT_MAX_TRIM_LEVEL.
|
* @group_id: The physical group ID from which to allocate JIT memory.
|
* Valid range is 0..(%MEMORY_GROUP_MANAGER_NR_GROUPS-1).
|
* @phys_pages_limit: Maximum number of physical pages to use to back the JIT
|
* region. Must not exceed @jit_va_pages.
|
*
|
* Return: 0 if success, negative error code otherwise.
|
*/
|
int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages,
|
int max_allocations, int trim_level, int group_id,
|
u64 phys_pages_limit);
|
|
/**
|
* kbase_region_tracker_init_exec - Initialize the GPU-executable memory region
|
* @kctx: kbase context
|
* @exec_va_pages: Size of the JIT region in pages.
|
* It must not be greater than 4 GB.
|
*
|
* Return: 0 if success, negative error code otherwise.
|
*/
|
int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages);
|
|
/**
|
* kbase_region_tracker_term - Terminate the JIT region
|
* @kctx: kbase context
|
*/
|
void kbase_region_tracker_term(struct kbase_context *kctx);
|
|
/**
|
* kbase_region_tracker_term_rbtree - Free memory for a region tracker
|
*
|
* @rbtree: Region tracker tree root
|
*
|
* This will free all the regions within the region tracker
|
*/
|
void kbase_region_tracker_term_rbtree(struct rb_root *rbtree);
|
|
struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(
|
struct kbase_context *kctx, u64 gpu_addr);
|
struct kbase_va_region *kbase_find_region_enclosing_address(
|
struct rb_root *rbtree, u64 gpu_addr);
|
|
/**
|
* kbase_region_tracker_find_region_base_address - Check that a pointer is
|
* actually a valid region.
|
* @kctx: kbase context containing the region
|
* @gpu_addr: pointer to check
|
*
|
* Must be called with context lock held.
|
*
|
* Return: pointer to the valid region on success, NULL otherwise
|
*/
|
struct kbase_va_region *kbase_region_tracker_find_region_base_address(
|
struct kbase_context *kctx, u64 gpu_addr);
|
struct kbase_va_region *kbase_find_region_base_address(struct rb_root *rbtree,
|
u64 gpu_addr);
|
|
struct kbase_va_region *kbase_alloc_free_region(struct kbase_device *kbdev, struct rb_root *rbtree,
|
u64 start_pfn, size_t nr_pages, int zone);
|
void kbase_free_alloced_region(struct kbase_va_region *reg);
|
int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg,
|
u64 addr, size_t nr_pages, size_t align);
|
int kbase_add_va_region_rbtree(struct kbase_device *kbdev,
|
struct kbase_va_region *reg, u64 addr, size_t nr_pages,
|
size_t align);
|
|
bool kbase_check_alloc_flags(unsigned long flags);
|
bool kbase_check_import_flags(unsigned long flags);
|
|
static inline bool kbase_import_size_is_valid(struct kbase_device *kbdev, u64 va_pages)
|
{
|
if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) {
|
dev_dbg(
|
kbdev->dev,
|
"Import attempted with va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!",
|
(unsigned long long)va_pages);
|
return false;
|
}
|
|
return true;
|
}
|
|
static inline bool kbase_alias_size_is_valid(struct kbase_device *kbdev, u64 va_pages)
|
{
|
if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) {
|
dev_dbg(
|
kbdev->dev,
|
"Alias attempted with va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!",
|
(unsigned long long)va_pages);
|
return false;
|
}
|
|
return true;
|
}
|
|
/**
|
* kbase_check_alloc_sizes - check user space sizes parameters for an
|
* allocation
|
*
|
* @kctx: kbase context
|
* @flags: The flags passed from user space
|
* @va_pages: The size of the requested region, in pages.
|
* @commit_pages: Number of pages to commit initially.
|
* @extension: Number of pages to grow by on GPU page fault and/or alignment
|
* (depending on flags)
|
*
|
* Makes checks on the size parameters passed in from user space for a memory
|
* allocation call, with respect to the flags requested.
|
*
|
* Return: 0 if sizes are valid for these flags, negative error code otherwise
|
*/
|
int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags,
|
u64 va_pages, u64 commit_pages, u64 extension);
|
|
/**
|
* kbase_update_region_flags - Convert user space flags to kernel region flags
|
*
|
* @kctx: kbase context
|
* @reg: The region to update the flags on
|
* @flags: The flags passed from user space
|
*
|
* The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and
|
* this function will fail if the system does not support system coherency.
|
*
|
* Return: 0 if successful, -EINVAL if the flags are not supported
|
*/
|
int kbase_update_region_flags(struct kbase_context *kctx,
|
struct kbase_va_region *reg, unsigned long flags);
|
|
/**
|
* kbase_gpu_vm_lock() - Acquire the per-context region list lock
|
* @kctx: KBase context
|
*
|
* Care must be taken when making an allocation whilst holding this lock, because of interaction
|
* with the Kernel's OoM-killer and use of this lock in &vm_operations_struct close() handlers.
|
*
|
* If this lock is taken during a syscall, and/or the allocation is 'small' then it is safe to use.
|
*
|
* If the caller is not in a syscall, and the allocation is 'large', then it must not hold this
|
* lock.
|
*
|
* This is because the kernel OoM killer might target the process corresponding to that same kbase
|
* context, and attempt to call the context's close() handlers for its open VMAs. This is safe if
|
* the allocating caller is in a syscall, because the VMA close() handlers are delayed until all
|
* syscalls have finished (noting that no new syscalls can start as the remaining user threads will
|
* have been killed too), and so there is no possibility of contention between the thread
|
* allocating with this lock held, and the VMA close() handler.
|
*
|
* However, outside of a syscall (e.g. a kworker or other kthread), one of kbase's VMA close()
|
* handlers (kbase_cpu_vm_close()) also takes this lock, and so prevents the process from being
|
* killed until the caller of the function allocating memory has released this lock. On subsequent
|
* retries for allocating a page, the OoM killer would be re-invoked but skips over the process
|
* stuck in its close() handler.
|
*
|
* Also because the caller is not in a syscall, the page allocation code in the kernel is not aware
|
* that the allocation is being done on behalf of another process, and so does not realize that
|
* process has received a kill signal due to an OoM, and so will continually retry with the OoM
|
* killer until enough memory has been released, or until all other killable processes have been
|
* killed (at which point the kernel halts with a panic).
|
*
|
* However, if the allocation outside of a syscall is small enough to be satisfied by killing
|
* another process, then the allocation completes, the caller releases this lock, and
|
* kbase_cpu_vm_close() can unblock and allow the process to be killed.
|
*
|
* Hence, this is effectively a deadlock with kbase_cpu_vm_close(), except that if the memory
|
* allocation is small enough the deadlock can be resolved. For that reason, such a memory deadlock
|
* is NOT discovered with CONFIG_PROVE_LOCKING.
|
*
|
* If this may be called outside of a syscall, consider moving allocations outside of this lock, or
|
* use __GFP_NORETRY for such allocations (which will allow direct-reclaim attempts, but will
|
* prevent OoM kills to satisfy the allocation, and will just fail the allocation instead).
|
*/
|
void kbase_gpu_vm_lock(struct kbase_context *kctx);
|
|
/**
|
* kbase_gpu_vm_unlock() - Release the per-context region list lock
|
* @kctx: KBase context
|
*/
|
void kbase_gpu_vm_unlock(struct kbase_context *kctx);
|
|
int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size);
|
|
/**
|
* kbase_gpu_mmap - Register region and map it on the GPU.
|
*
|
* @kctx: kbase context containing the region
|
* @reg: the region to add
|
* @addr: the address to insert the region at
|
* @nr_pages: the number of pages in the region
|
* @align: the minimum alignment in pages
|
* @mmu_sync_info: Indicates whether this call is synchronous wrt MMU ops.
|
*
|
* Call kbase_add_va_region() and map the region on the GPU.
|
*
|
* Return: 0 on success, error code otherwise.
|
*/
|
int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg,
|
u64 addr, size_t nr_pages, size_t align,
|
enum kbase_caller_mmu_sync_info mmu_sync_info);
|
|
/**
|
* kbase_gpu_munmap - Remove the region from the GPU and unregister it.
|
*
|
* @kctx: KBase context
|
* @reg: The region to remove
|
*
|
* Must be called with context lock held.
|
*
|
* Return: 0 on success, error code otherwise.
|
*/
|
int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg);
|
|
/**
|
* kbase_mmu_update - Configure an address space on the GPU to the specified
|
* MMU tables
|
*
|
* @kbdev: Kbase device structure
|
* @mmut: The set of MMU tables to be configured on the address space
|
* @as_nr: The address space to be configured
|
*
|
* The caller has the following locking conditions:
|
* - It must hold kbase_device->mmu_hw_mutex
|
* - It must hold the hwaccess_lock
|
*/
|
void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut,
|
int as_nr);
|
|
/**
|
* kbase_mmu_disable() - Disable the MMU for a previously active kbase context.
|
* @kctx: Kbase context
|
*
|
* Disable and perform the required cache maintenance to remove the all
|
* data from provided kbase context from the GPU caches.
|
*
|
* The caller has the following locking conditions:
|
* - It must hold kbase_device->mmu_hw_mutex
|
* - It must hold the hwaccess_lock
|
*/
|
void kbase_mmu_disable(struct kbase_context *kctx);
|
|
/**
|
* kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified
|
* address space.
|
* @kbdev: Kbase device
|
* @as_nr: The address space number to set to unmapped.
|
*
|
* This function must only be called during reset/power-up and it used to
|
* ensure the registers are in a known state.
|
*
|
* The caller must hold kbdev->mmu_hw_mutex.
|
*/
|
void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr);
|
|
void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
|
|
#if defined(CONFIG_MALI_VECTOR_DUMP)
|
/**
|
* kbase_mmu_dump() - Dump the MMU tables to a buffer.
|
*
|
* @kctx: The kbase context to dump
|
* @nr_pages: The number of pages to allocate for the buffer.
|
*
|
* This function allocates a buffer (of @c nr_pages pages) to hold a dump
|
* of the MMU tables and fills it. If the buffer is too small
|
* then the return value will be NULL.
|
*
|
* The GPU vm lock must be held when calling this function.
|
*
|
* The buffer returned should be freed with @ref vfree when it is no longer
|
* required.
|
*
|
* Return: The address of the buffer containing the MMU dump or NULL on error
|
* (including if the @c nr_pages is too small)
|
*/
|
void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages);
|
#endif
|
|
/**
|
* kbase_sync_now - Perform cache maintenance on a memory region
|
*
|
* @kctx: The kbase context of the region
|
* @sset: A syncset structure describing the region and direction of the
|
* synchronisation required
|
*
|
* Return: 0 on success or error code
|
*/
|
int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset);
|
void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa,
|
struct tagged_addr gpu_pa, off_t offset, size_t size,
|
enum kbase_sync_type sync_fn);
|
|
/* OS specific functions */
|
int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr);
|
int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg);
|
void kbase_os_mem_map_lock(struct kbase_context *kctx);
|
void kbase_os_mem_map_unlock(struct kbase_context *kctx);
|
|
/**
|
* kbasep_os_process_page_usage_update() - Update the memory allocation
|
* counters for the current process.
|
*
|
* @kctx: The kbase context
|
* @pages: The desired delta to apply to the memory usage counters.
|
*
|
* OS specific call to updates the current memory allocation counters
|
* for the current process with the supplied delta.
|
*/
|
|
void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages);
|
|
/**
|
* kbase_process_page_usage_inc() - Add to the memory allocation counters for
|
* the current process
|
*
|
* @kctx: The kernel base context used for the allocation.
|
* @pages: The desired delta to apply to the memory usage counters.
|
*
|
* OS specific call to add to the current memory allocation counters for
|
* the current process by the supplied amount.
|
*/
|
|
static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages)
|
{
|
kbasep_os_process_page_usage_update(kctx, pages);
|
}
|
|
/**
|
* kbase_process_page_usage_dec() - Subtract from the memory allocation
|
* counters for the current process.
|
*
|
* @kctx: The kernel base context used for the allocation.
|
* @pages: The desired delta to apply to the memory usage counters.
|
*
|
* OS specific call to subtract from the current memory allocation counters
|
* for the current process by the supplied amount.
|
*/
|
|
static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages)
|
{
|
kbasep_os_process_page_usage_update(kctx, 0 - pages);
|
}
|
|
/**
|
* kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU
|
* mapping of a memory allocation containing a given address range
|
*
|
* @kctx: The kernel base context used for the allocation.
|
* @uaddr: Start of the CPU virtual address range.
|
* @size: Size of the CPU virtual address range (in bytes).
|
* @offset: The offset from the start of the allocation to the specified CPU
|
* virtual address.
|
*
|
* Searches for a CPU mapping of any part of any region that fully encloses the
|
* CPU virtual address range specified by @uaddr and @size. Returns a failure
|
* indication if only part of the address range lies within a CPU mapping.
|
*
|
* Return: 0 if offset was obtained successfully. Error code otherwise.
|
*/
|
int kbasep_find_enclosing_cpu_mapping_offset(
|
struct kbase_context *kctx,
|
unsigned long uaddr, size_t size, u64 *offset);
|
|
/**
|
* kbasep_find_enclosing_gpu_mapping_start_and_offset() - Find the address of
|
* the start of GPU virtual memory region which encloses @gpu_addr for the
|
* @size length in bytes
|
*
|
* @kctx: The kernel base context within which the memory is searched.
|
* @gpu_addr: GPU virtual address for which the region is sought; defines
|
* the beginning of the provided region.
|
* @size: The length (in bytes) of the provided region for which the
|
* GPU virtual memory region is sought.
|
* @start: Pointer to the location where the address of the start of
|
* the found GPU virtual memory region is.
|
* @offset: Pointer to the location where the offset of @gpu_addr into
|
* the found GPU virtual memory region is.
|
*
|
* Searches for the memory region in GPU virtual memory space which contains
|
* the region defined by the @gpu_addr and @size, where @gpu_addr is the
|
* beginning and @size the length in bytes of the provided region. If found,
|
* the location of the start address of the GPU virtual memory region is
|
* passed in @start pointer and the location of the offset of the region into
|
* the GPU virtual memory region is passed in @offset pointer.
|
*
|
* Return: 0 on success, error code otherwise.
|
*/
|
int kbasep_find_enclosing_gpu_mapping_start_and_offset(
|
struct kbase_context *kctx,
|
u64 gpu_addr, size_t size, u64 *start, u64 *offset);
|
|
/**
|
* kbase_alloc_phy_pages_helper - Allocates physical pages.
|
* @alloc: allocation object to add pages to
|
* @nr_pages_requested: number of physical pages to allocate
|
*
|
* Allocates @nr_pages_requested and updates the alloc object.
|
*
|
* Note: if kbase_gpu_vm_lock() is to be held around this function to ensure thread-safe updating
|
* of @alloc, then refer to the documentation of kbase_gpu_vm_lock() about the requirements of
|
* either calling during a syscall, or ensuring the allocation is small. These requirements prevent
|
* an effective deadlock between the kernel's OoM killer and kbase's VMA close() handlers, which
|
* could take kbase_gpu_vm_lock() too.
|
*
|
* If the requirements of kbase_gpu_vm_lock() cannot be satisfied when calling this function, but
|
* @alloc must still be updated in a thread-safe way, then instead use
|
* kbase_alloc_phy_pages_helper_locked() and restructure callers into the sequence outlined there.
|
*
|
* This function cannot be used from interrupt context
|
*
|
* Return: 0 if all pages have been successfully allocated. Error code otherwise
|
*/
|
int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc,
|
size_t nr_pages_requested);
|
|
/**
|
* kbase_alloc_phy_pages_helper_locked - Allocates physical pages.
|
* @alloc: allocation object to add pages to
|
* @pool: Memory pool to allocate from
|
* @nr_pages_requested: number of physical pages to allocate
|
*
|
* @prealloc_sa: Information about the partial allocation if the amount of memory requested
|
* is not a multiple of 2MB. One instance of struct kbase_sub_alloc must be
|
* allocated by the caller if kbdev->pagesize_2mb is enabled.
|
*
|
* Allocates @nr_pages_requested and updates the alloc object. This function does not allocate new
|
* pages from the kernel, and therefore will never trigger the OoM killer. Therefore, it can be
|
* called whilst a thread operating outside of a syscall has held the region list lock
|
* (kbase_gpu_vm_lock()), as it will not cause an effective deadlock with VMA close() handlers used
|
* by the OoM killer.
|
*
|
* As new pages can not be allocated, the caller must ensure there are sufficient pages in the
|
* pool. Usage of this function should look like :
|
*
|
* kbase_gpu_vm_lock(kctx);
|
* kbase_mem_pool_lock(pool)
|
* while (kbase_mem_pool_size(pool) < pages_required) {
|
* kbase_mem_pool_unlock(pool)
|
* kbase_gpu_vm_unlock(kctx);
|
* kbase_mem_pool_grow(pool)
|
* kbase_gpu_vm_lock(kctx);
|
* kbase_mem_pool_lock(pool)
|
* }
|
* kbase_alloc_phy_pages_helper_locked(pool)
|
* kbase_mem_pool_unlock(pool)
|
* // Perform other processing that requires vm_lock...
|
* kbase_gpu_vm_unlock(kctx);
|
*
|
* This ensures that the pool can be grown to the required size and that the allocation can
|
* complete without another thread using the newly grown pages.
|
*
|
* If kbdev->pagesize_2mb is enabled and the allocation is >= 2MB, then @pool must be one of the
|
* pools from alloc->imported.native.kctx->mem_pools.large[]. Otherwise it must be one of the
|
* mempools from alloc->imported.native.kctx->mem_pools.small[].
|
*
|
* @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be pre-allocated because we
|
* must not sleep (due to the usage of kmalloc()) whilst holding pool->pool_lock. @prealloc_sa
|
* shall be set to NULL if it has been consumed by this function to indicate that the caller no
|
* longer owns it and should not access it further.
|
*
|
* Note: Caller must hold @pool->pool_lock
|
*
|
* Return: Pointer to array of allocated pages. NULL on failure.
|
*/
|
struct tagged_addr *kbase_alloc_phy_pages_helper_locked(
|
struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool,
|
size_t nr_pages_requested,
|
struct kbase_sub_alloc **prealloc_sa);
|
|
/**
|
* kbase_free_phy_pages_helper() - Free physical pages.
|
*
|
* @alloc: allocation object to free pages from
|
* @nr_pages_to_free: number of physical pages to free
|
*
|
* Free @nr_pages_to_free pages and updates the alloc object.
|
*
|
* Return: 0 on success, otherwise a negative error code
|
*/
|
int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free);
|
|
/**
|
* kbase_free_phy_pages_helper_locked - Free pages allocated with
|
* kbase_alloc_phy_pages_helper_locked()
|
* @alloc: Allocation object to free pages from
|
* @pool: Memory pool to return freed pages to
|
* @pages: Pages allocated by kbase_alloc_phy_pages_helper_locked()
|
* @nr_pages_to_free: Number of physical pages to free
|
*
|
* This function atomically frees pages allocated with
|
* kbase_alloc_phy_pages_helper_locked(). @pages is the pointer to the page
|
* array that is returned by that function. @pool must be the pool that the
|
* pages were originally allocated from.
|
*
|
* If the mem_pool has been unlocked since the allocation then
|
* kbase_free_phy_pages_helper() should be used instead.
|
*/
|
void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc,
|
struct kbase_mem_pool *pool, struct tagged_addr *pages,
|
size_t nr_pages_to_free);
|
|
static inline void kbase_set_dma_addr_as_priv(struct page *p, dma_addr_t dma_addr)
|
{
|
SetPagePrivate(p);
|
if (sizeof(dma_addr_t) > sizeof(p->private)) {
|
/* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the
|
* private field stays the same. So we have to be clever and
|
* use the fact that we only store DMA addresses of whole pages,
|
* so the low bits should be zero
|
*/
|
KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1)));
|
set_page_private(p, dma_addr >> PAGE_SHIFT);
|
} else {
|
set_page_private(p, dma_addr);
|
}
|
}
|
|
static inline dma_addr_t kbase_dma_addr_as_priv(struct page *p)
|
{
|
if (sizeof(dma_addr_t) > sizeof(p->private))
|
return ((dma_addr_t)page_private(p)) << PAGE_SHIFT;
|
|
return (dma_addr_t)page_private(p);
|
}
|
|
static inline void kbase_clear_dma_addr_as_priv(struct page *p)
|
{
|
ClearPagePrivate(p);
|
}
|
|
static inline struct kbase_page_metadata *kbase_page_private(struct page *p)
|
{
|
return (struct kbase_page_metadata *)page_private(p);
|
}
|
|
static inline dma_addr_t kbase_dma_addr(struct page *p)
|
{
|
if (kbase_page_migration_enabled)
|
return kbase_page_private(p)->dma_addr;
|
|
return kbase_dma_addr_as_priv(p);
|
}
|
|
static inline dma_addr_t kbase_dma_addr_from_tagged(struct tagged_addr tagged_pa)
|
{
|
phys_addr_t pa = as_phys_addr_t(tagged_pa);
|
struct page *page = pfn_to_page(PFN_DOWN(pa));
|
dma_addr_t dma_addr =
|
is_huge(tagged_pa) ? kbase_dma_addr_as_priv(page) : kbase_dma_addr(page);
|
|
return dma_addr;
|
}
|
|
/**
|
* kbase_flush_mmu_wqs() - Flush MMU workqueues.
|
* @kbdev: Device pointer.
|
*
|
* This function will cause any outstanding page or bus faults to be processed.
|
* It should be called prior to powering off the GPU.
|
*/
|
void kbase_flush_mmu_wqs(struct kbase_device *kbdev);
|
|
/**
|
* kbase_sync_single_for_device - update physical memory and give GPU ownership
|
* @kbdev: Device pointer
|
* @handle: DMA address of region
|
* @size: Size of region to sync
|
* @dir: DMA data direction
|
*/
|
|
void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle,
|
size_t size, enum dma_data_direction dir);
|
|
/**
|
* kbase_sync_single_for_cpu - update physical memory and give CPU ownership
|
* @kbdev: Device pointer
|
* @handle: DMA address of region
|
* @size: Size of region to sync
|
* @dir: DMA data direction
|
*/
|
|
void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle,
|
size_t size, enum dma_data_direction dir);
|
|
#if IS_ENABLED(CONFIG_DEBUG_FS)
|
/**
|
* kbase_jit_debugfs_init - Add per context debugfs entry for JIT.
|
* @kctx: kbase context
|
*/
|
void kbase_jit_debugfs_init(struct kbase_context *kctx);
|
#endif /* CONFIG_DEBUG_FS */
|
|
/**
|
* kbase_jit_init - Initialize the JIT memory pool management
|
* @kctx: kbase context
|
*
|
* Return: zero on success or negative error number on failure.
|
*/
|
int kbase_jit_init(struct kbase_context *kctx);
|
|
/**
|
* kbase_jit_allocate - Allocate JIT memory
|
* @kctx: kbase context
|
* @info: JIT allocation information
|
* @ignore_pressure_limit: Whether the JIT memory pressure limit is ignored
|
*
|
* Return: JIT allocation on success or NULL on failure.
|
*/
|
struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
|
const struct base_jit_alloc_info *info,
|
bool ignore_pressure_limit);
|
|
/**
|
* kbase_jit_free - Free a JIT allocation
|
* @kctx: kbase context
|
* @reg: JIT allocation
|
*
|
* Frees a JIT allocation and places it into the free pool for later reuse.
|
*/
|
void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg);
|
|
/**
|
* kbase_jit_backing_lost - Inform JIT that an allocation has lost backing
|
* @reg: JIT allocation
|
*/
|
void kbase_jit_backing_lost(struct kbase_va_region *reg);
|
|
/**
|
* kbase_jit_evict - Evict a JIT allocation from the pool
|
* @kctx: kbase context
|
*
|
* Evict the least recently used JIT allocation from the pool. This can be
|
* required if normal VA allocations are failing due to VA exhaustion.
|
*
|
* Return: True if a JIT allocation was freed, false otherwise.
|
*/
|
bool kbase_jit_evict(struct kbase_context *kctx);
|
|
/**
|
* kbase_jit_term - Terminate the JIT memory pool management
|
* @kctx: kbase context
|
*/
|
void kbase_jit_term(struct kbase_context *kctx);
|
|
#if MALI_JIT_PRESSURE_LIMIT_BASE
|
/**
|
* kbase_trace_jit_report_gpu_mem_trace_enabled - variant of
|
* kbase_trace_jit_report_gpu_mem() that should only be called once the
|
* corresponding tracepoint is verified to be enabled
|
* @kctx: kbase context
|
* @reg: Just-in-time memory region to trace
|
* @flags: combination of values from enum kbase_jit_report_flags
|
*/
|
void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx,
|
struct kbase_va_region *reg, unsigned int flags);
|
#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
|
|
/**
|
* kbase_trace_jit_report_gpu_mem - Trace information about the GPU memory used
|
* to make a JIT report
|
* @kctx: kbase context
|
* @reg: Just-in-time memory region to trace
|
* @flags: combination of values from enum kbase_jit_report_flags
|
*
|
* Information is traced using the trace_mali_jit_report_gpu_mem() tracepoint.
|
*
|
* In case that tracepoint is not enabled, this function should have the same
|
* low overheads as a tracepoint itself (i.e. use of 'jump labels' to avoid
|
* conditional branches)
|
*
|
* This can take the reg_lock on @kctx, do not use in places where this lock is
|
* already held.
|
*
|
* Note: this has to be a macro because at this stage the tracepoints have not
|
* been included. Also gives no opportunity for the compiler to mess up
|
* inlining it.
|
*/
|
#if MALI_JIT_PRESSURE_LIMIT_BASE
|
#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \
|
do { \
|
if (trace_mali_jit_report_gpu_mem_enabled()) \
|
kbase_trace_jit_report_gpu_mem_trace_enabled( \
|
(kctx), (reg), (flags)); \
|
} while (0)
|
#else
|
#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \
|
CSTD_NOP(kctx, reg, flags)
|
#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
|
|
#if MALI_JIT_PRESSURE_LIMIT_BASE
|
/**
|
* kbase_jit_report_update_pressure - safely update the JIT physical page
|
* pressure and JIT region's estimate of used_pages
|
* @kctx: kbase context, to update the current physical pressure
|
* @reg: Just-in-time memory region to update with @new_used_pages
|
* @new_used_pages: new value of number of pages used in the JIT region
|
* @flags: combination of values from enum kbase_jit_report_flags
|
*
|
* Takes care of:
|
* - correctly updating the pressure given the current reg->used_pages and
|
* new_used_pages
|
* - then updating the %kbase_va_region used_pages member
|
*
|
* Precondition:
|
* - new_used_pages <= reg->nr_pages
|
*/
|
void kbase_jit_report_update_pressure(struct kbase_context *kctx,
|
struct kbase_va_region *reg, u64 new_used_pages,
|
unsigned int flags);
|
|
/**
|
* kbase_jit_trim_necessary_pages() - calculate and trim the least pages
|
* possible to satisfy a new JIT allocation
|
*
|
* @kctx: Pointer to the kbase context
|
* @needed_pages: Number of JIT physical pages by which trimming is requested.
|
* The actual number of pages trimmed could differ.
|
*
|
* Before allocating a new just-in-time memory region or reusing a previous
|
* one, ensure that the total JIT physical page usage also will not exceed the
|
* pressure limit.
|
*
|
* If there are no reported-on allocations, then we already guarantee this will
|
* be the case - because our current pressure then only comes from the va_pages
|
* of each JIT region, hence JIT physical page usage is guaranteed to be
|
* bounded by this.
|
*
|
* However as soon as JIT allocations become "reported on", the pressure is
|
* lowered to allow new JIT regions to be allocated. It is after such a point
|
* that the total JIT physical page usage could (either now or in the future on
|
* a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly
|
* allocated JIT regions. Hence, trim any "reported on" regions.
|
*
|
* Any pages freed will go into the pool and be allocated from there in
|
* kbase_mem_alloc().
|
*/
|
void kbase_jit_trim_necessary_pages(struct kbase_context *kctx,
|
size_t needed_pages);
|
|
/*
|
* Same as kbase_jit_request_phys_increase(), except that Caller is supposed
|
* to take jit_evict_lock also on @kctx before calling this function.
|
*/
|
static inline void
|
kbase_jit_request_phys_increase_locked(struct kbase_context *kctx,
|
size_t needed_pages)
|
{
|
#if !MALI_USE_CSF
|
lockdep_assert_held(&kctx->jctx.lock);
|
#endif /* !MALI_USE_CSF */
|
lockdep_assert_held(&kctx->reg_lock);
|
lockdep_assert_held(&kctx->jit_evict_lock);
|
|
kctx->jit_phys_pages_to_be_allocated += needed_pages;
|
|
kbase_jit_trim_necessary_pages(kctx,
|
kctx->jit_phys_pages_to_be_allocated);
|
}
|
|
/**
|
* kbase_jit_request_phys_increase() - Increment the backing pages count and do
|
* the required trimming before allocating pages for a JIT allocation.
|
*
|
* @kctx: Pointer to the kbase context
|
* @needed_pages: Number of pages to be allocated for the JIT allocation.
|
*
|
* This function needs to be called before allocating backing pages for a
|
* just-in-time memory region. The backing pages are currently allocated when,
|
*
|
* - A new JIT region is created.
|
* - An old JIT region is reused from the cached pool.
|
* - GPU page fault occurs for the active JIT region.
|
* - Backing is grown for the JIT region through the commit ioctl.
|
*
|
* This function would ensure that the total JIT physical page usage does not
|
* exceed the pressure limit even when the backing pages get allocated
|
* simultaneously for multiple JIT allocations from different threads.
|
*
|
* There should be a matching call to kbase_jit_done_phys_increase(), after
|
* the pages have been allocated and accounted against the active JIT
|
* allocation.
|
*
|
* Caller is supposed to take reg_lock on @kctx before calling this function.
|
*/
|
static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx,
|
size_t needed_pages)
|
{
|
#if !MALI_USE_CSF
|
lockdep_assert_held(&kctx->jctx.lock);
|
#endif /* !MALI_USE_CSF */
|
lockdep_assert_held(&kctx->reg_lock);
|
|
mutex_lock(&kctx->jit_evict_lock);
|
kbase_jit_request_phys_increase_locked(kctx, needed_pages);
|
mutex_unlock(&kctx->jit_evict_lock);
|
}
|
|
/**
|
* kbase_jit_done_phys_increase() - Decrement the backing pages count after the
|
* allocation of pages for a JIT allocation.
|
*
|
* @kctx: Pointer to the kbase context
|
* @needed_pages: Number of pages that were allocated for the JIT allocation.
|
*
|
* This function should be called after backing pages have been allocated and
|
* accounted against the active JIT allocation.
|
* The call should be made when the following have been satisfied:
|
* when the allocation is on the jit_active_head.
|
* when additional needed_pages have been allocated.
|
* kctx->reg_lock was held during the above and has not yet been unlocked.
|
* Failure to call this function before unlocking the kctx->reg_lock when
|
* either the above have changed may result in over-accounting the memory.
|
* This ensures kbase_jit_trim_necessary_pages() gets a consistent count of
|
* the memory.
|
*
|
* A matching call to kbase_jit_request_phys_increase() should have been made,
|
* before the allocation of backing pages.
|
*
|
* Caller is supposed to take reg_lock on @kctx before calling this function.
|
*/
|
static inline void kbase_jit_done_phys_increase(struct kbase_context *kctx,
|
size_t needed_pages)
|
{
|
lockdep_assert_held(&kctx->reg_lock);
|
|
WARN_ON(kctx->jit_phys_pages_to_be_allocated < needed_pages);
|
|
kctx->jit_phys_pages_to_be_allocated -= needed_pages;
|
}
|
#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
|
|
/**
|
* kbase_has_exec_va_zone - EXEC_VA zone predicate
|
*
|
* @kctx: kbase context
|
*
|
* Determine whether an EXEC_VA zone has been created for the GPU address space
|
* of the given kbase context.
|
*
|
* Return: True if the kbase context has an EXEC_VA zone.
|
*/
|
bool kbase_has_exec_va_zone(struct kbase_context *kctx);
|
|
/**
|
* kbase_map_external_resource - Map an external resource to the GPU.
|
* @kctx: kbase context.
|
* @reg: External resource to map.
|
* @locked_mm: The mm_struct which has been locked for this operation.
|
*
|
* On successful mapping, the VA region and the gpu_alloc refcounts will be
|
* increased, making it safe to use and store both values directly.
|
*
|
* Return: Zero on success, or negative error code.
|
*/
|
int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_region *reg,
|
struct mm_struct *locked_mm);
|
|
/**
|
* kbase_unmap_external_resource - Unmap an external resource from the GPU.
|
* @kctx: kbase context.
|
* @reg: VA region corresponding to external resource
|
*
|
* On successful unmapping, the VA region and the gpu_alloc refcounts will
|
* be decreased. If the refcount reaches zero, both @reg and the corresponding
|
* allocation may be freed, so using them after returning from this function
|
* requires the caller to explicitly check their state.
|
*/
|
void kbase_unmap_external_resource(struct kbase_context *kctx, struct kbase_va_region *reg);
|
|
/**
|
* kbase_unpin_user_buf_page - Unpin a page of a user buffer.
|
* @page: page to unpin
|
*
|
* The caller must have ensured that there are no CPU mappings for @page (as
|
* might be created from the struct kbase_mem_phy_alloc that tracks @page), and
|
* that userspace will not be able to recreate the CPU mappings again.
|
*/
|
void kbase_unpin_user_buf_page(struct page *page);
|
|
/**
|
* kbase_jd_user_buf_pin_pages - Pin the pages of a user buffer.
|
* @kctx: kbase context.
|
* @reg: The region associated with the imported user buffer.
|
*
|
* To successfully pin the pages for a user buffer the current mm_struct must
|
* be the same as the mm_struct of the user buffer. After successfully pinning
|
* the pages further calls to this function succeed without doing work.
|
*
|
* Return: zero on success or negative number on failure.
|
*/
|
int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx,
|
struct kbase_va_region *reg);
|
|
/**
|
* kbase_sticky_resource_init - Initialize sticky resource management.
|
* @kctx: kbase context
|
*
|
* Return: zero on success or negative error number on failure.
|
*/
|
int kbase_sticky_resource_init(struct kbase_context *kctx);
|
|
/**
|
* kbase_sticky_resource_acquire - Acquire a reference on a sticky resource.
|
* @kctx: kbase context.
|
* @gpu_addr: The GPU address of the external resource.
|
*
|
* Return: The metadata object which represents the binding between the
|
* external resource and the kbase context on success or NULL on failure.
|
*/
|
struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(
|
struct kbase_context *kctx, u64 gpu_addr);
|
|
/**
|
* kbase_sticky_resource_release - Release a reference on a sticky resource.
|
* @kctx: kbase context.
|
* @meta: Binding metadata.
|
* @gpu_addr: GPU address of the external resource.
|
*
|
* If meta is NULL then gpu_addr will be used to scan the metadata list and
|
* find the matching metadata (if any), otherwise the provided meta will be
|
* used and gpu_addr will be ignored.
|
*
|
* Return: True if the release found the metadata and the reference was dropped.
|
*/
|
bool kbase_sticky_resource_release(struct kbase_context *kctx,
|
struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr);
|
|
/**
|
* kbase_sticky_resource_release_force - Release a sticky resource.
|
* @kctx: kbase context.
|
* @meta: Binding metadata.
|
* @gpu_addr: GPU address of the external resource.
|
*
|
* If meta is NULL then gpu_addr will be used to scan the metadata list and
|
* find the matching metadata (if any), otherwise the provided meta will be
|
* used and gpu_addr will be ignored.
|
*
|
* Return: True if the release found the metadata and the resource was
|
* released.
|
*/
|
bool kbase_sticky_resource_release_force(struct kbase_context *kctx,
|
struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr);
|
|
/**
|
* kbase_sticky_resource_term - Terminate sticky resource management.
|
* @kctx: kbase context
|
*/
|
void kbase_sticky_resource_term(struct kbase_context *kctx);
|
|
/**
|
* kbase_mem_pool_lock - Lock a memory pool
|
* @pool: Memory pool to lock
|
*/
|
static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool)
|
{
|
spin_lock(&pool->pool_lock);
|
}
|
|
/**
|
* kbase_mem_pool_unlock - Release a memory pool
|
* @pool: Memory pool to lock
|
*/
|
static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool)
|
{
|
spin_unlock(&pool->pool_lock);
|
}
|
|
/**
|
* kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable.
|
* @alloc: The physical allocation
|
*/
|
void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc);
|
|
#if MALI_USE_CSF
|
/**
|
* kbase_link_event_mem_page - Add the new event memory region to the per
|
* context list of event pages.
|
* @kctx: Pointer to kbase context
|
* @reg: Pointer to the region allocated for event memory.
|
*
|
* The region being linked shouldn't have been marked as free and should
|
* have KBASE_REG_CSF_EVENT flag set for it.
|
*/
|
static inline void kbase_link_event_mem_page(struct kbase_context *kctx,
|
struct kbase_va_region *reg)
|
{
|
lockdep_assert_held(&kctx->reg_lock);
|
|
WARN_ON(reg->flags & KBASE_REG_FREE);
|
WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT));
|
|
list_add(®->link, &kctx->csf.event_pages_head);
|
}
|
|
/**
|
* kbase_unlink_event_mem_page - Remove the event memory region from the per
|
* context list of event pages.
|
* @kctx: Pointer to kbase context
|
* @reg: Pointer to the region allocated for event memory.
|
*
|
* The region being un-linked shouldn't have been marked as free and should
|
* have KBASE_REG_CSF_EVENT flag set for it.
|
*/
|
static inline void kbase_unlink_event_mem_page(struct kbase_context *kctx,
|
struct kbase_va_region *reg)
|
{
|
lockdep_assert_held(&kctx->reg_lock);
|
|
WARN_ON(reg->flags & KBASE_REG_FREE);
|
WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT));
|
|
list_del(®->link);
|
}
|
|
/**
|
* kbase_mcu_shared_interface_region_tracker_init - Initialize the rb tree to
|
* manage the shared interface segment of MCU firmware address space.
|
* @kbdev: Pointer to the kbase device
|
*
|
* Return: zero on success or negative error number on failure.
|
*/
|
int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev);
|
|
/**
|
* kbase_mcu_shared_interface_region_tracker_term - Teardown the rb tree
|
* managing the shared interface segment of MCU firmware address space.
|
* @kbdev: Pointer to the kbase device
|
*/
|
void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev);
|
#endif
|
|
/**
|
* kbase_mem_umm_map - Map dma-buf
|
* @kctx: Pointer to the kbase context
|
* @reg: Pointer to the region of the imported dma-buf to map
|
*
|
* Map a dma-buf on the GPU. The mappings are reference counted.
|
*
|
* Return: 0 on success, or a negative error code.
|
*/
|
int kbase_mem_umm_map(struct kbase_context *kctx,
|
struct kbase_va_region *reg);
|
|
/**
|
* kbase_mem_umm_unmap - Unmap dma-buf
|
* @kctx: Pointer to the kbase context
|
* @reg: Pointer to the region of the imported dma-buf to unmap
|
* @alloc: Pointer to the alloc to release
|
*
|
* Unmap a dma-buf from the GPU. The mappings are reference counted.
|
*
|
* @reg must be the original region with GPU mapping of @alloc; or NULL. If
|
* @reg is NULL, or doesn't match @alloc, the GPU page table entries matching
|
* @reg will not be updated.
|
*
|
* @alloc must be a valid physical allocation of type
|
* KBASE_MEM_TYPE_IMPORTED_UMM that was previously mapped by
|
* kbase_mem_umm_map(). The dma-buf attachment referenced by @alloc will
|
* release it's mapping reference, and if the refcount reaches 0, also be
|
* unmapped, regardless of the value of @reg.
|
*/
|
void kbase_mem_umm_unmap(struct kbase_context *kctx,
|
struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc);
|
|
/**
|
* kbase_mem_do_sync_imported - Sync caches for imported memory
|
* @kctx: Pointer to the kbase context
|
* @reg: Pointer to the region with imported memory to sync
|
* @sync_fn: The type of sync operation to perform
|
*
|
* Sync CPU caches for supported (currently only dma-buf (UMM)) memory.
|
* Attempting to sync unsupported imported memory types will result in an error
|
* code, -EINVAL.
|
*
|
* Return: 0 on success, or a negative error code.
|
*/
|
int kbase_mem_do_sync_imported(struct kbase_context *kctx,
|
struct kbase_va_region *reg, enum kbase_sync_type sync_fn);
|
|
/**
|
* kbase_mem_copy_to_pinned_user_pages - Memcpy from source input page to
|
* an unaligned address at a given offset from the start of a target page.
|
*
|
* @dest_pages: Pointer to the array of pages to which the content is
|
* to be copied from the provided @src_page.
|
* @src_page: Pointer to the page which correspond to the source page
|
* from which the copying will take place.
|
* @to_copy: Total number of bytes pending to be copied from
|
* @src_page to @target_page_nr within @dest_pages.
|
* This will get decremented by number of bytes we
|
* managed to copy from source page to target pages.
|
* @nr_pages: Total number of pages present in @dest_pages.
|
* @target_page_nr: Target page number to which @src_page needs to be
|
* copied. This will get incremented by one if
|
* we are successful in copying from source page.
|
* @offset: Offset in bytes into the target pages from which the
|
* copying is to be performed.
|
*
|
* Return: 0 on success, or a negative error code.
|
*/
|
int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages,
|
void *src_page, size_t *to_copy, unsigned int nr_pages,
|
unsigned int *target_page_nr, size_t offset);
|
|
/**
|
* kbase_reg_zone_end_pfn - return the end Page Frame Number of @zone
|
* @zone: zone to query
|
*
|
* Return: The end of the zone corresponding to @zone
|
*/
|
static inline u64 kbase_reg_zone_end_pfn(struct kbase_reg_zone *zone)
|
{
|
return zone->base_pfn + zone->va_size_pages;
|
}
|
|
/**
|
* kbase_ctx_reg_zone_init - initialize a zone in @kctx
|
* @kctx: Pointer to kbase context
|
* @zone_bits: A KBASE_REG_ZONE_<...> to initialize
|
* @base_pfn: Page Frame Number in GPU virtual address space for the start of
|
* the Zone
|
* @va_size_pages: Size of the Zone in pages
|
*/
|
static inline void kbase_ctx_reg_zone_init(struct kbase_context *kctx,
|
unsigned long zone_bits,
|
u64 base_pfn, u64 va_size_pages)
|
{
|
struct kbase_reg_zone *zone;
|
|
lockdep_assert_held(&kctx->reg_lock);
|
WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
|
|
zone = &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
|
*zone = (struct kbase_reg_zone){
|
.base_pfn = base_pfn, .va_size_pages = va_size_pages,
|
};
|
}
|
|
/**
|
* kbase_ctx_reg_zone_get_nolock - get a zone from @kctx where the caller does
|
* not have @kctx 's region lock
|
* @kctx: Pointer to kbase context
|
* @zone_bits: A KBASE_REG_ZONE_<...> to retrieve
|
*
|
* This should only be used in performance-critical paths where the code is
|
* resilient to a race with the zone changing.
|
*
|
* Return: The zone corresponding to @zone_bits
|
*/
|
static inline struct kbase_reg_zone *
|
kbase_ctx_reg_zone_get_nolock(struct kbase_context *kctx,
|
unsigned long zone_bits)
|
{
|
WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
|
|
return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
|
}
|
|
/**
|
* kbase_ctx_reg_zone_get - get a zone from @kctx
|
* @kctx: Pointer to kbase context
|
* @zone_bits: A KBASE_REG_ZONE_<...> to retrieve
|
*
|
* The get is not refcounted - there is no corresponding 'put' operation
|
*
|
* Return: The zone corresponding to @zone_bits
|
*/
|
static inline struct kbase_reg_zone *
|
kbase_ctx_reg_zone_get(struct kbase_context *kctx, unsigned long zone_bits)
|
{
|
lockdep_assert_held(&kctx->reg_lock);
|
WARN_ON(!kbase_is_ctx_reg_zone(zone_bits));
|
|
return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)];
|
}
|
|
/**
|
* kbase_mem_allow_alloc - Check if allocation of GPU memory is allowed
|
* @kctx: Pointer to kbase context
|
*
|
* Don't allow the allocation of GPU memory if the ioctl has been issued
|
* from the forked child process using the mali device file fd inherited from
|
* the parent process.
|
*
|
* Return: true if allocation is allowed.
|
*/
|
static inline bool kbase_mem_allow_alloc(struct kbase_context *kctx)
|
{
|
return (kctx->process_mm == current->mm);
|
}
|
|
/**
|
* kbase_mem_mmgrab - Wrapper function to take reference on mm_struct of current process
|
*/
|
static inline void kbase_mem_mmgrab(void)
|
{
|
/* This merely takes a reference on the memory descriptor structure
|
* i.e. mm_struct of current process and not on its address space and
|
* so won't block the freeing of address space on process exit.
|
*/
|
#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
|
atomic_inc(¤t->mm->mm_count);
|
#else
|
mmgrab(current->mm);
|
#endif
|
}
|
|
/**
|
* kbase_mem_group_id_get - Get group ID from flags
|
* @flags: Flags to pass to base_mem_alloc
|
*
|
* This inline function extracts the encoded group ID from flags
|
* and converts it into numeric value (0~15).
|
*
|
* Return: group ID(0~15) extracted from the parameter
|
*/
|
static inline int kbase_mem_group_id_get(base_mem_alloc_flags flags)
|
{
|
KBASE_DEBUG_ASSERT((flags & ~BASE_MEM_FLAGS_INPUT_MASK) == 0);
|
return (int)BASE_MEM_GROUP_ID_GET(flags);
|
}
|
|
/**
|
* kbase_mem_group_id_set - Set group ID into base_mem_alloc_flags
|
* @id: group ID(0~15) you want to encode
|
*
|
* This inline function encodes specific group ID into base_mem_alloc_flags.
|
* Parameter 'id' should lie in-between 0 to 15.
|
*
|
* Return: base_mem_alloc_flags with the group ID (id) encoded
|
*
|
* The return value can be combined with other flags against base_mem_alloc
|
* to identify a specific memory group.
|
*/
|
static inline base_mem_alloc_flags kbase_mem_group_id_set(int id)
|
{
|
return BASE_MEM_GROUP_ID_SET(id);
|
}
|
#endif /* _KBASE_MEM_H_ */
|
