/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
|
/*
|
*
|
* (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
/*
|
* Definitions (types, defines, etcs) specific to Job Manager Kbase.
|
* They are placed here to allow the hierarchy of header files to work.
|
*/
|
|
#ifndef _KBASE_JM_DEFS_H_
|
#define _KBASE_JM_DEFS_H_
|
|
#include "mali_kbase_js_defs.h"
|
|
/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */
|
#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1
|
|
/*
|
* Number of milliseconds before resetting the GPU when a job cannot be "zapped"
|
* from the hardware. Note that the time is actually
|
* ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and
|
* the GPU actually being reset to give other contexts time for their jobs
|
* to be soft-stopped and removed from the hardware before resetting.
|
*/
|
#define ZAP_TIMEOUT 1000
|
|
/*
|
* Prevent soft-stops from occurring in scheduling situations
|
*
|
* This is not due to HW issues, but when scheduling is desired to be more
|
* predictable.
|
*
|
* Therefore, soft stop may still be disabled due to HW issues.
|
*
|
* Soft stop will still be used for non-scheduling purposes e.g. when
|
* terminating a context.
|
*
|
* if not in use, define this value to 0 instead of being undefined.
|
*/
|
#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0
|
|
/*
|
* Prevent hard-stops from occurring in scheduling situations
|
*
|
* This is not due to HW issues, but when scheduling is desired to be more
|
* predictable.
|
*
|
* Hard stop will still be used for non-scheduling purposes e.g. when
|
* terminating a context.
|
*
|
* if not in use, define this value to 0 instead of being undefined.
|
*/
|
#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0
|
|
/* Atom has been previously soft-stopped */
|
#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1)
|
/* Atom has been previously retried to execute */
|
#define KBASE_KATOM_FLAGS_RERUN (1<<2)
|
/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps
|
* to disambiguate short-running job chains during soft/hard stopping of jobs
|
*/
|
#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)
|
/* Atom has been previously hard-stopped. */
|
#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
|
/* Atom has caused us to enter disjoint state */
|
#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
|
/* Atom blocked on cross-slot dependency */
|
#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
|
/* Atom has fail dependency on cross-slot dependency */
|
#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
|
/* Atom is currently in the list of atoms blocked on cross-slot dependencies */
|
#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9)
|
/* Atom is currently holding a context reference */
|
#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10)
|
/* Atom requires GPU to be in protected mode */
|
#define KBASE_KATOM_FLAG_PROTECTED (1<<11)
|
/* Atom has been stored in runnable_tree */
|
#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12)
|
/* Atom is waiting for L2 caches to power up in order to enter protected mode */
|
#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13)
|
|
/* SW related flags about types of JS_COMMAND action
|
* NOTE: These must be masked off by JS_COMMAND_MASK
|
*/
|
|
/* This command causes a disjoint event */
|
#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100
|
|
/* Bitmask of all SW related flags */
|
#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT)
|
|
#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
|
#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \
|
"Must update JS_COMMAND_SW_<..> bitmasks"
|
#endif
|
|
/* Soft-stop command that causes a Disjoint event. This of course isn't
|
* entirely masked off by JS_COMMAND_MASK
|
*/
|
#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
|
(JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
|
|
#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
|
|
/* Serialize atoms within a slot (ie only one atom per job slot) */
|
#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0)
|
/* Serialize atoms between slots (ie only one job slot running at any time) */
|
#define KBASE_SERIALIZE_INTER_SLOT (1 << 1)
|
/* Reset the GPU after each atom completion */
|
#define KBASE_SERIALIZE_RESET (1 << 2)
|
|
#if IS_ENABLED(CONFIG_DEBUG_FS)
|
/**
|
* struct base_job_fault_event - keeps track of the atom which faulted or which
|
* completed after the faulty atom but before the
|
* debug data for faulty atom was dumped.
|
*
|
* @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for
|
* the atom which faulted.
|
* @katom: pointer to the atom for which job fault occurred or which
|
* completed after the faulty atom.
|
* @job_fault_work: work item, queued only for the faulty atom, which waits for
|
* the dumping to get completed and then does the bottom half
|
* of job done for the atoms which followed the faulty atom.
|
* @head: List head used to store the atom in the global list of
|
* faulty atoms or context specific list of atoms which got
|
* completed during the dump.
|
* @reg_offset: offset of the register to be dumped next, only applicable
|
* for the faulty atom.
|
*/
|
struct base_job_fault_event {
|
|
u32 event_code;
|
struct kbase_jd_atom *katom;
|
struct work_struct job_fault_work;
|
struct list_head head;
|
int reg_offset;
|
};
|
#endif
|
|
/**
|
* struct kbase_jd_atom_dependency - Contains the dependency info for an atom.
|
* @atom: pointer to the dependee atom.
|
* @dep_type: type of dependency on the dependee @atom, i.e. order or data
|
* dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency.
|
*/
|
struct kbase_jd_atom_dependency {
|
struct kbase_jd_atom *atom;
|
u8 dep_type;
|
};
|
|
/**
|
* kbase_jd_katom_dep_atom - Retrieves a read-only reference to the
|
* dependee atom.
|
* @dep: pointer to the dependency info structure.
|
*
|
* Return: readonly reference to dependee atom.
|
*/
|
static inline const struct kbase_jd_atom *
|
kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
|
{
|
LOCAL_ASSERT(dep != NULL);
|
|
return (const struct kbase_jd_atom *)(dep->atom);
|
}
|
|
/**
|
* kbase_jd_katom_dep_type - Retrieves the dependency type info
|
*
|
* @dep: pointer to the dependency info structure.
|
*
|
* Return: the type of dependency there is on the dependee atom.
|
*/
|
static inline u8 kbase_jd_katom_dep_type(
|
const struct kbase_jd_atom_dependency *dep)
|
{
|
LOCAL_ASSERT(dep != NULL);
|
|
return dep->dep_type;
|
}
|
|
/**
|
* kbase_jd_katom_dep_set - sets up the dependency info structure
|
* as per the values passed.
|
* @const_dep: pointer to the dependency info structure to be setup.
|
* @a: pointer to the dependee atom.
|
* @type: type of dependency there is on the dependee atom.
|
*/
|
static inline void kbase_jd_katom_dep_set(
|
const struct kbase_jd_atom_dependency *const_dep,
|
struct kbase_jd_atom *a, u8 type)
|
{
|
struct kbase_jd_atom_dependency *dep;
|
|
LOCAL_ASSERT(const_dep != NULL);
|
|
dep = (struct kbase_jd_atom_dependency *)const_dep;
|
|
dep->atom = a;
|
dep->dep_type = type;
|
}
|
|
/**
|
* kbase_jd_katom_dep_clear - resets the dependency info structure
|
*
|
* @const_dep: pointer to the dependency info structure to be setup.
|
*/
|
static inline void kbase_jd_katom_dep_clear(
|
const struct kbase_jd_atom_dependency *const_dep)
|
{
|
struct kbase_jd_atom_dependency *dep;
|
|
LOCAL_ASSERT(const_dep != NULL);
|
|
dep = (struct kbase_jd_atom_dependency *)const_dep;
|
|
dep->atom = NULL;
|
dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
|
}
|
|
/**
|
* enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it
|
* becomes runnable, with respect to job slot
|
* ringbuffer/fifo.
|
* @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo,
|
* which implies that either atom has not become
|
* runnable due to dependency or has completed
|
* the execution on GPU.
|
* @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is
|
* blocked due to cross slot dependency,
|
* can't be submitted to GPU.
|
* @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot
|
* fifo but is waiting for the completion of
|
* previously added atoms in current & other
|
* slots, as their protected mode requirements
|
* do not match with the current atom.
|
* @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo
|
* and is waiting for completion of protected
|
* mode transition, needed before the atom is
|
* submitted to GPU.
|
* @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is
|
* waiting for the cores, which are needed to
|
* execute the job chain represented by the atom,
|
* to become available
|
* @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to
|
* GPU.
|
* @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted
|
* to GPU.
|
* @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some
|
* failure, but only after the previously added
|
* atoms in fifo have completed or have also
|
* been returned to JS.
|
*/
|
enum kbase_atom_gpu_rb_state {
|
KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
|
KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
|
KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV,
|
KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION,
|
KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
|
KBASE_ATOM_GPU_RB_READY,
|
KBASE_ATOM_GPU_RB_SUBMITTED,
|
KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1
|
};
|
|
/**
|
* enum kbase_atom_enter_protected_state - The state of an atom with respect to
|
* the preparation for GPU's entry into protected mode,
|
* becomes pertinent only after atom's state with respect
|
* to slot ringbuffer is
|
* KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
|
* @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any
|
* atoms currently submitted to GPU and protected mode
|
* transition is not already in progress.
|
* @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to
|
* become disabled before entry into protected mode.
|
* @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
|
* preparation for the coherency change. L2 shall be
|
* powered down and GPU shall come out of fully
|
* coherent mode before entering protected mode.
|
* @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change;
|
* for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on
|
* so that coherency register contains correct value when
|
* GPU enters protected mode.
|
* @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for
|
* BASE_HW_ISSUE_TGOX_R1_1234 check
|
* that L2 is powered up and switch GPU to protected mode.
|
*/
|
enum kbase_atom_enter_protected_state {
|
/*
|
* NOTE: The integer value of this must match
|
* KBASE_ATOM_EXIT_PROTECTED_CHECK.
|
*/
|
KBASE_ATOM_ENTER_PROTECTED_CHECK = 0,
|
KBASE_ATOM_ENTER_PROTECTED_HWCNT,
|
KBASE_ATOM_ENTER_PROTECTED_IDLE_L2,
|
KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY,
|
KBASE_ATOM_ENTER_PROTECTED_FINISHED,
|
};
|
|
/**
|
* enum kbase_atom_exit_protected_state - The state of an atom with respect to
|
* the preparation for GPU's exit from protected mode,
|
* becomes pertinent only after atom's state with respect
|
* to slot ngbuffer is
|
* KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
|
* @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any
|
* atoms currently submitted to GPU and protected mode
|
* transition is not already in progress.
|
* @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
|
* preparation for the reset, as exiting protected mode
|
* requires a reset.
|
* @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from
|
* protected mode
|
* @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to
|
* complete
|
*/
|
enum kbase_atom_exit_protected_state {
|
/*
|
* NOTE: The integer value of this must match
|
* KBASE_ATOM_ENTER_PROTECTED_CHECK.
|
*/
|
KBASE_ATOM_EXIT_PROTECTED_CHECK = 0,
|
KBASE_ATOM_EXIT_PROTECTED_IDLE_L2,
|
KBASE_ATOM_EXIT_PROTECTED_RESET,
|
KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT,
|
};
|
|
/**
|
* struct kbase_ext_res - Contains the info for external resources referred
|
* by an atom, which have been mapped on GPU side.
|
* @gpu_address: Start address of the memory region allocated for
|
* the resource from GPU virtual address space.
|
* @alloc: pointer to physical pages tracking object, set on
|
* mapping the external resource on GPU side.
|
*/
|
struct kbase_ext_res {
|
u64 gpu_address;
|
struct kbase_mem_phy_alloc *alloc;
|
};
|
|
/**
|
* struct kbase_jd_atom - object representing the atom, containing the complete
|
* state and attributes of an atom.
|
* @work: work item for the bottom half processing of the atom,
|
* by JD or JS, after it got executed on GPU or the
|
* input fence got signaled
|
* @start_timestamp: time at which the atom was submitted to the GPU, by
|
* updating the JS_HEAD_NEXTn register.
|
* @udata: copy of the user data sent for the atom in
|
* base_jd_submit.
|
* @kctx: Pointer to the base context with which the atom is
|
* associated.
|
* @dep_head: Array of 2 list heads, pointing to the two list of
|
* atoms
|
* which are blocked due to dependency on this atom.
|
* @dep_item: Array of 2 list heads, used to store the atom in the
|
* list of other atoms depending on the same dependee
|
* atom.
|
* @dep: Array containing the dependency info for the 2 atoms
|
* on which the atom depends upon.
|
* @jd_item: List head used during job dispatch job_done
|
* processing - as dependencies may not be entirely
|
* resolved at this point,
|
* we need to use a separate list head.
|
* @in_jd_list: flag set to true if atom's @jd_item is currently on
|
* a list, prevents atom being processed twice.
|
* @jit_ids: Zero-terminated array of IDs of just-in-time memory
|
* allocations written to by the atom. When the atom
|
* completes, the value stored at the
|
* &struct_base_jit_alloc_info.heap_info_gpu_addr of
|
* each allocation is read in order to enforce an
|
* overall physical memory usage limit.
|
* @nr_extres: number of external resources referenced by the atom.
|
* @extres: pointer to the location containing info about
|
* @nr_extres external resources referenced by the atom.
|
* @device_nr: indicates the coregroup with which the atom is
|
* associated, when
|
* BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified.
|
* @jc: GPU address of the job-chain.
|
* @softjob_data: Copy of data read from the user space buffer that @jc
|
* points to.
|
* @fence: Stores either an input or output sync fence,
|
* depending on soft-job type
|
* @sync_waiter: Pointer to the sync fence waiter structure passed to
|
* the callback function on signaling of the input
|
* fence.
|
* @dma_fence: object containing pointers to both input & output
|
* fences and other related members used for explicit
|
* sync through soft jobs and for the implicit
|
* synchronization required on access to external
|
* resources.
|
* @dma_fence.fence_in: Input fence
|
* @dma_fence.fence: Points to the dma-buf output fence for this atom.
|
* @dma_fence.context: The dma-buf fence context number for this atom. A
|
* unique context number is allocated to each katom in
|
* the context on context creation.
|
* @dma_fence.seqno: The dma-buf fence sequence number for this atom. This
|
* is increased every time this katom uses dma-buf fence
|
* @dma_fence.callbacks: List of all callbacks set up to wait on other fences
|
* @dma_fence.dep_count: Atomic counter of number of outstandind dma-buf fence
|
* dependencies for this atom.
|
* @event_code: Event code for the job chain represented by the atom,
|
* both HW and low-level SW events are represented by
|
* event codes.
|
* @core_req: bitmask of BASE_JD_REQ_* flags specifying either
|
* Hw or Sw requirements for the job chain represented
|
* by the atom.
|
* @ticks: Number of scheduling ticks for which atom has been
|
* running on the GPU.
|
* @sched_priority: Priority of the atom for Job scheduling, as per the
|
* KBASE_JS_ATOM_SCHED_PRIO_*.
|
* @completed: Wait queue to wait upon for the completion of atom.
|
* @status: Indicates at high level at what stage the atom is in,
|
* as per KBASE_JD_ATOM_STATE_*, that whether it is not
|
* in use or its queued in JD or given to JS or
|
* submitted to Hw or it completed the execution on Hw.
|
* @work_id: used for GPU tracepoints, its a snapshot of the
|
* 'work_id' counter in kbase_jd_context which is
|
* incremented on every call to base_jd_submit.
|
* @slot_nr: Job slot chosen for the atom.
|
* @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the
|
* excat low level state of the atom.
|
* @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely
|
* tracking atom's state after it has entered
|
* Job scheduler on becoming runnable. Atom
|
* could be blocked due to cross slot dependency
|
* or waiting for the shader cores to become available
|
* or waiting for protected mode transitions to
|
* complete.
|
* @need_cache_flush_cores_retained: flag indicating that manual flush of GPU
|
* cache is needed for the atom and the shader cores
|
* used for atom have been kept on.
|
* @blocked: flag indicating that atom's resubmission to GPU is
|
* blocked till the work item is scheduled to return the
|
* atom to JS.
|
* @seq_nr: user-space sequence number, to order atoms in some
|
* temporal order
|
* @pre_dep: Pointer to atom that this atom has same-slot
|
* dependency on
|
* @post_dep: Pointer to atom that has same-slot dependency on
|
* this atom
|
* @x_pre_dep: Pointer to atom that this atom has cross-slot
|
* dependency on
|
* @x_post_dep: Pointer to atom that has cross-slot dependency on
|
* this atom
|
* @flush_id: The GPU's flush count recorded at the time of
|
* submission,
|
* used for the cache flush optimization
|
* @fault_event: Info for dumping the debug data on Job fault.
|
* @queue: List head used for 4 different purposes :
|
* Adds atom to the list of dma-buf fence waiting atoms.
|
* Adds atom to the list of atoms blocked due to cross
|
* slot dependency.
|
* Adds atom to the list of softjob atoms for which JIT
|
* allocation has been deferred
|
* Adds atom to the list of softjob atoms waiting for
|
* the signaling of fence.
|
* @jit_node: Used to keep track of all JIT free/alloc jobs in
|
* submission order
|
* @jit_blocked: Flag indicating that JIT allocation requested through
|
* softjob atom will be reattempted after the impending
|
* free of other active JIT allocations.
|
* @will_fail_event_code: If non-zero, this indicates that the atom will fail
|
* with the set event_code when the atom is processed.
|
* Used for special handling of atoms, which have a data
|
* dependency on the failed atoms.
|
* @protected_state: State of the atom, as per
|
* KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*,
|
* when transitioning into or out of protected mode.
|
* Atom will be either entering or exiting the
|
* protected mode.
|
* @protected_state.enter: entering the protected mode.
|
* @protected_state.exit: exiting the protected mode.
|
* @runnable_tree_node: The node added to context's job slot specific rb tree
|
* when the atom becomes runnable.
|
* @age: Age of atom relative to other atoms in the context,
|
* is snapshot of the age_count counter in kbase
|
* context.
|
* @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified.
|
* @renderpass_id:Renderpass identifier used to associate an atom that has
|
* BASE_JD_REQ_START_RENDERPASS set in its core requirements
|
* with an atom that has BASE_JD_REQ_END_RENDERPASS set.
|
* @jc_fragment: Set of GPU fragment job chains
|
* @retry_count: TODO: Not used,to be removed
|
*/
|
struct kbase_jd_atom {
|
struct work_struct work;
|
ktime_t start_timestamp;
|
|
struct base_jd_udata udata;
|
struct kbase_context *kctx;
|
|
struct list_head dep_head[2];
|
struct list_head dep_item[2];
|
const struct kbase_jd_atom_dependency dep[2];
|
struct list_head jd_item;
|
bool in_jd_list;
|
|
#if MALI_JIT_PRESSURE_LIMIT_BASE
|
u8 jit_ids[2];
|
#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
|
|
u16 nr_extres;
|
struct kbase_ext_res *extres;
|
|
u32 device_nr;
|
u64 jc;
|
void *softjob_data;
|
#if defined(CONFIG_SYNC)
|
struct sync_fence *fence;
|
struct sync_fence_waiter sync_waiter;
|
#endif /* CONFIG_SYNC */
|
#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
|
struct {
|
/* Use the functions/API defined in mali_kbase_fence.h to
|
* when working with this sub struct
|
*/
|
#if defined(CONFIG_SYNC_FILE)
|
#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
|
struct fence *fence_in;
|
#else
|
struct dma_fence *fence_in;
|
#endif
|
#endif
|
/* This points to the dma-buf output fence for this atom. If
|
* this is NULL then there is no fence for this atom and the
|
* following fields related to dma_fence may have invalid data.
|
*
|
* The context and seqno fields contain the details for this
|
* fence.
|
*
|
* This fence is signaled when the katom is completed,
|
* regardless of the event_code of the katom (signal also on
|
* failure).
|
*/
|
#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
|
struct fence *fence;
|
#else
|
struct dma_fence *fence;
|
#endif
|
unsigned int context;
|
atomic_t seqno;
|
/* This contains a list of all callbacks set up to wait on
|
* other fences. This atom must be held back from JS until all
|
* these callbacks have been called and dep_count have reached
|
* 0. The initial value of dep_count must be equal to the
|
* number of callbacks on this list.
|
*
|
* This list is protected by jctx.lock. Callbacks are added to
|
* this list when the atom is built and the wait are set up.
|
* All the callbacks then stay on the list until all callbacks
|
* have been called and the atom is queued, or cancelled, and
|
* then all callbacks are taken off the list and freed.
|
*/
|
struct list_head callbacks;
|
/* Atomic counter of number of outstandind dma-buf fence
|
* dependencies for this atom. When dep_count reaches 0 the
|
* atom may be queued.
|
*
|
* The special value "-1" may only be set after the count
|
* reaches 0, while holding jctx.lock. This indicates that the
|
* atom has been handled, either queued in JS or cancelled.
|
*
|
* If anyone but the dma-fence worker sets this to -1 they must
|
* ensure that any potentially queued worker must have
|
* completed before allowing the atom to be marked as unused.
|
* This can be done by flushing the fence work queue:
|
* kctx->dma_fence.wq.
|
*/
|
atomic_t dep_count;
|
} dma_fence;
|
#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */
|
|
/* Note: refer to kbasep_js_atom_retained_state, which will take a copy
|
* of some of the following members
|
*/
|
enum base_jd_event_code event_code;
|
base_jd_core_req core_req;
|
u8 jobslot;
|
u8 renderpass_id;
|
struct base_jd_fragment jc_fragment;
|
|
u32 ticks;
|
int sched_priority;
|
|
wait_queue_head_t completed;
|
enum kbase_jd_atom_state status;
|
#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
|
int work_id;
|
#endif
|
int slot_nr;
|
|
u32 atom_flags;
|
|
int retry_count;
|
|
enum kbase_atom_gpu_rb_state gpu_rb_state;
|
|
bool need_cache_flush_cores_retained;
|
|
atomic_t blocked;
|
|
u64 seq_nr;
|
|
struct kbase_jd_atom *pre_dep;
|
struct kbase_jd_atom *post_dep;
|
|
struct kbase_jd_atom *x_pre_dep;
|
struct kbase_jd_atom *x_post_dep;
|
|
u32 flush_id;
|
|
#if IS_ENABLED(CONFIG_DEBUG_FS)
|
struct base_job_fault_event fault_event;
|
#endif
|
struct list_head queue;
|
|
struct list_head jit_node;
|
bool jit_blocked;
|
|
enum base_jd_event_code will_fail_event_code;
|
|
union {
|
enum kbase_atom_enter_protected_state enter;
|
enum kbase_atom_exit_protected_state exit;
|
} protected_state;
|
|
struct rb_node runnable_tree_node;
|
|
u32 age;
|
};
|
|
static inline bool kbase_jd_katom_is_protected(
|
const struct kbase_jd_atom *katom)
|
{
|
return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED);
|
}
|
|
/*
|
* Theory of operations:
|
*
|
* Atom objects are statically allocated within the context structure.
|
*
|
* Each atom is the head of two lists, one for the "left" set of dependencies,
|
* one for the "right" set.
|
*/
|
|
#define KBASE_JD_DEP_QUEUE_SIZE 256
|
|
/**
|
* enum kbase_jd_renderpass_state - State of a renderpass
|
* @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to
|
* START.
|
* @KBASE_JD_RP_START: Renderpass making a first attempt at tiling.
|
* Can transition to PEND_OOM or COMPLETE.
|
* @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much
|
* memory and has a soft-stop pending. Can transition to
|
* OOM or COMPLETE.
|
* @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much
|
* memory and therefore switched to incremental
|
* rendering. The fragment job chain is forced to run.
|
* Can only transition to RETRY.
|
* @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at
|
* tiling. Can transition to RETRY_PEND_OOM or COMPLETE.
|
* @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at
|
* tiling used too much memory again and has a
|
* soft-stop pending. Can transition to RETRY_OOM
|
* or COMPLETE.
|
* @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at
|
* tiling used too much memory again. The fragment job
|
* chain is forced to run. Can only transition to RETRY.
|
*
|
* A state machine is used to control incremental rendering.
|
*/
|
enum kbase_jd_renderpass_state {
|
KBASE_JD_RP_COMPLETE, /* COMPLETE => START */
|
KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */
|
KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */
|
KBASE_JD_RP_OOM, /* OOM => RETRY */
|
KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or
|
* COMPLETE
|
*/
|
KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or
|
* COMPLETE
|
*/
|
KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */
|
};
|
|
/**
|
* struct kbase_jd_renderpass - Data for a renderpass
|
* @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then
|
* all other members are invalid.
|
* Both the job dispatcher context and hwaccess_lock must be
|
* locked to modify this so that it can be read with either
|
* (or both) locked.
|
* @start_katom: Address of the atom that is the start of a renderpass.
|
* Both the job dispatcher context and hwaccess_lock must be
|
* locked to modify this so that it can be read with either
|
* (or both) locked.
|
* @end_katom: Address of the atom that is the end of a renderpass, or NULL
|
* if that atom hasn't been added to the job scheduler yet.
|
* The job dispatcher context and hwaccess_lock must be
|
* locked to modify this so that it can be read with either
|
* (or both) locked.
|
* @oom_reg_list: A list of region structures which triggered out-of-memory.
|
* The hwaccess_lock must be locked to access this.
|
*
|
* Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS
|
* are associated with an object of this type, which is created and maintained
|
* by kbase to keep track of each renderpass.
|
*/
|
struct kbase_jd_renderpass {
|
enum kbase_jd_renderpass_state state;
|
struct kbase_jd_atom *start_katom;
|
struct kbase_jd_atom *end_katom;
|
struct list_head oom_reg_list;
|
};
|
|
/**
|
* struct kbase_jd_context - per context object encapsulating all the
|
* Job dispatcher related state.
|
* @lock: lock to serialize the updates made to the
|
* Job dispatcher state and kbase_jd_atom objects.
|
* @sched_info: Structure encapsulating all the Job scheduling
|
* info.
|
* @atoms: Array of the objects representing atoms,
|
* containing the complete state and attributes
|
* of an atom.
|
* @renderpasses: Array of renderpass state for incremental
|
* rendering, indexed by user-specified renderpass
|
* ID.
|
* @job_nr: Tracks the number of atoms being processed by the
|
* kbase. This includes atoms that are not tracked by
|
* scheduler: 'not ready to run' & 'dependency-only'
|
* jobs.
|
* @zero_jobs_wait: Waitq that reflects whether there are no jobs
|
* (including SW-only dependency jobs). This is set
|
* when no jobs are present on the ctx, and clear
|
* when there are jobs.
|
* This must be updated atomically with @job_nr.
|
* note: Job Dispatcher knows about more jobs than
|
* the Job Scheduler as it is unaware of jobs that
|
* are blocked on dependencies and SW-only dependency
|
* jobs. This waitq can be waited upon to find out
|
* when the context jobs are all done/cancelled
|
* (including those that might've been blocked
|
* on dependencies) - and so, whether it can be
|
* terminated. However, it should only be terminated
|
* once it is not present in the run-pool.
|
* Since the waitq is only set under @lock,
|
* the waiter should also briefly obtain and drop
|
* @lock to guarantee that the setter has completed
|
* its work on the kbase_context
|
* @job_done_wq: Workqueue to which the per atom work item is
|
* queued for bottom half processing when the
|
* atom completes
|
* execution on GPU or the input fence get signaled.
|
* @tb_lock: Lock to serialize the write access made to @tb to
|
* to store the register access trace messages.
|
* @tb: Pointer to the Userspace accessible buffer storing
|
* the trace messages for register read/write
|
* accesses made by the Kbase. The buffer is filled
|
* in circular fashion.
|
* @tb_wrap_offset: Offset to the end location in the trace buffer,
|
* the write pointer is moved to the beginning on
|
* reaching this offset.
|
* @work_id: atomic variable used for GPU tracepoints,
|
* incremented on every call to base_jd_submit.
|
* @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
|
* allocate & free, in submission order, protected
|
* by kbase_jd_context.lock.
|
* @jit_pending_alloc: A list of just-in-time memory allocation
|
* soft-jobs which will be reattempted after the
|
* impending free of other active allocations.
|
* @max_priority: Max priority level allowed for this context.
|
*/
|
struct kbase_jd_context {
|
struct mutex lock;
|
struct kbasep_js_kctx_info sched_info;
|
struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];
|
struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT];
|
struct workqueue_struct *job_done_wq;
|
|
wait_queue_head_t zero_jobs_wait;
|
spinlock_t tb_lock;
|
u32 *tb;
|
u32 job_nr;
|
size_t tb_wrap_offset;
|
|
#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
|
atomic_t work_id;
|
#endif
|
|
struct list_head jit_atoms_head;
|
struct list_head jit_pending_alloc;
|
int max_priority;
|
};
|
|
/**
|
* struct jsctx_queue - JS context atom queue
|
* @runnable_tree: Root of RB-tree containing currently runnable atoms on this
|
* job slot.
|
* @x_dep_head: Head item of the linked list of atoms blocked on cross-slot
|
* dependencies. Atoms on this list will be moved to the
|
* runnable_tree when the blocking atom completes.
|
*
|
* hwaccess_lock must be held when accessing this structure.
|
*/
|
struct jsctx_queue {
|
struct rb_root runnable_tree;
|
struct list_head x_dep_head;
|
};
|
|
/**
|
* struct kbase_as - Object representing an address space of GPU.
|
* @number: Index at which this address space structure is present
|
* in an array of address space structures embedded inside
|
* the &struct kbase_device.
|
* @pf_wq: Workqueue for processing work items related to
|
* Page fault and Bus fault handling.
|
* @work_pagefault: Work item for the Page fault handling.
|
* @work_busfault: Work item for the Bus fault handling.
|
* @pf_data: Data relating to Page fault.
|
* @bf_data: Data relating to Bus fault.
|
* @current_setup: Stores the MMU configuration for this address space.
|
*/
|
struct kbase_as {
|
int number;
|
struct workqueue_struct *pf_wq;
|
struct work_struct work_pagefault;
|
struct work_struct work_busfault;
|
struct kbase_fault pf_data;
|
struct kbase_fault bf_data;
|
struct kbase_mmu_setup current_setup;
|
};
|
|
#endif /* _KBASE_JM_DEFS_H_ */
|
