From 1543e317f1da31b75942316931e8f491a8920811 Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Thu, 04 Jan 2024 10:08:02 +0000
Subject: [PATCH] disable FB
---
kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 580 +++++++++++++++++++++++++++++++++++++++++++++++++--------
1 files changed, 496 insertions(+), 84 deletions(-)
diff --git a/kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h
index 53526ce..6fa0e27 100644
--- a/kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h
+++ b/kernel/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
*
- * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -30,6 +30,13 @@
#include <linux/wait.h>
#include "mali_kbase_csf_firmware.h"
+#include "mali_kbase_refcount_defs.h"
+#include "mali_kbase_csf_event.h"
+#include <uapi/gpu/arm/bifrost/csf/mali_kbase_csf_errors_dumpfault.h>
+
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
+#include <debug/backend/mali_kbase_debug_coresight_internal_csf.h>
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
/* Maximum number of KCPU command queues to be created per GPU address space.
*/
@@ -54,7 +61,7 @@
#define CSF_FIRMWARE_ENTRY_ZERO (1ul << 31)
/**
- * enum kbase_csf_bind_state - bind state of the queue
+ * enum kbase_csf_queue_bind_state - bind state of the queue
*
* @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link
* between queue and the group to which it was bound or being bound is removed.
@@ -219,11 +226,19 @@
* management reference. This can happen if the GPU
* becomes idle for a duration exceeding a threshold,
* or due to a system triggered suspend action.
+ * @SCHED_SLEEPING: The scheduler is in low-power mode with scheduling
+ * operations suspended and is not holding the power
+ * management reference. This state is set, only for the
+ * GPUs that supports the sleep feature, when GPU idle
+ * notification is received. The state is changed to
+ * @SCHED_SUSPENDED from the runtime suspend callback
+ * function after the suspend of CSGs.
*/
enum kbase_csf_scheduler_state {
SCHED_BUSY,
SCHED_INACTIVE,
SCHED_SUSPENDED,
+ SCHED_SLEEPING,
};
/**
@@ -243,6 +258,36 @@
KBASE_QUEUE_GROUP_PRIORITY_COUNT
};
+/**
+ * enum kbase_timeout_selector - The choice of which timeout to get scaled
+ * using the lowest GPU frequency.
+ * @CSF_FIRMWARE_TIMEOUT: Response timeout from CSF firmware.
+ * @CSF_PM_TIMEOUT: Timeout for GPU Power Management to reach the desired
+ * Shader, L2 and MCU state.
+ * @CSF_GPU_RESET_TIMEOUT: Waiting timeout for GPU reset to complete.
+ * @CSF_CSG_SUSPEND_TIMEOUT: Timeout given for all active CSGs to be suspended.
+ * @CSF_FIRMWARE_BOOT_TIMEOUT: Maximum time to wait for firmware to boot.
+ * @CSF_FIRMWARE_PING_TIMEOUT: Maximum time to wait for firmware to respond
+ * to a ping from KBase.
+ * @CSF_SCHED_PROTM_PROGRESS_TIMEOUT: Timeout used to prevent protected mode execution hang.
+ * @MMU_AS_INACTIVE_WAIT_TIMEOUT: Maximum waiting time in ms for the completion
+ * of a MMU operation
+ * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. Must be last in
+ * the enum.
+ */
+enum kbase_timeout_selector {
+ CSF_FIRMWARE_TIMEOUT,
+ CSF_PM_TIMEOUT,
+ CSF_GPU_RESET_TIMEOUT,
+ CSF_CSG_SUSPEND_TIMEOUT,
+ CSF_FIRMWARE_BOOT_TIMEOUT,
+ CSF_FIRMWARE_PING_TIMEOUT,
+ CSF_SCHED_PROTM_PROGRESS_TIMEOUT,
+ MMU_AS_INACTIVE_WAIT_TIMEOUT,
+
+ /* Must be the last in the enum */
+ KBASE_TIMEOUT_SELECTOR_COUNT
+};
/**
* struct kbase_csf_notification - Event or error generated as part of command
@@ -261,9 +306,9 @@
*
* @kctx: Pointer to the base context with which this GPU command queue
* is associated.
- * @reg: Pointer to the region allocated from the shared
- * interface segment for mapping the User mode
- * input/output pages in MCU firmware address space.
+ * @user_io_gpu_va: The start GPU VA address of this queue's userio pages. Only
+ * valid (i.e. not 0 ) when the queue is enabled and its owner
+ * group has a runtime bound csg_reg (group region).
* @phys: Pointer to the physical pages allocated for the
* pair or User mode input/output page
* @user_io_addr: Pointer to the permanent kernel mapping of User mode
@@ -319,21 +364,34 @@
* @trace_buffer_size: CS trace buffer size for the queue.
* @trace_cfg: CS trace configuration parameters.
* @error: GPU command queue fatal information to pass to user space.
- * @fatal_event_work: Work item to handle the CS fatal event reported for this
- * queue.
- * @cs_fatal_info: Records additional information about the CS fatal event.
- * @cs_fatal: Records information about the CS fatal event.
+ * @cs_error_work: Work item to handle the CS fatal event reported for this
+ * queue or the CS fault event if dump on fault is enabled
+ * and acknowledgment for CS fault event needs to be done
+ * after dumping is complete.
+ * @cs_error_info: Records additional information about the CS fatal event or
+ * about CS fault event if dump on fault is enabled.
+ * @cs_error: Records information about the CS fatal event or
+ * about CS fault event if dump on fault is enabled.
+ * @cs_error_fatal: Flag to track if the CS fault or CS fatal event occurred.
+ * @pending: Indicating whether the queue has new submitted work.
+ * @extract_ofs: The current EXTRACT offset, this is only updated when handling
+ * the GLB IDLE IRQ if the idle timeout value is non-0 in order
+ * to help detect a queue's true idle status.
+ * @saved_cmd_ptr: The command pointer value for the GPU queue, saved when the
+ * group to which queue is bound is suspended.
+ * This can be useful in certain cases to know that till which
+ * point the execution reached in the Linear command buffer.
*/
struct kbase_queue {
struct kbase_context *kctx;
- struct kbase_va_region *reg;
+ u64 user_io_gpu_va;
struct tagged_addr phys[2];
char *user_io_addr;
u64 handle;
int doorbell_nr;
unsigned long db_file_offset;
struct list_head link;
- atomic_t refcount;
+ kbase_refcount_t refcount;
struct kbase_queue_group *group;
struct kbase_va_region *queue_reg;
struct work_struct oom_event_work;
@@ -353,34 +411,47 @@
u32 trace_buffer_size;
u32 trace_cfg;
struct kbase_csf_notification error;
- struct work_struct fatal_event_work;
- u64 cs_fatal_info;
- u32 cs_fatal;
+ struct work_struct cs_error_work;
+ u64 cs_error_info;
+ u32 cs_error;
+ bool cs_error_fatal;
+ atomic_t pending;
+ u64 extract_ofs;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+ u64 saved_cmd_ptr;
+#endif /* CONFIG_DEBUG_FS */
};
/**
* struct kbase_normal_suspend_buffer - Object representing a normal
* suspend buffer for queue group.
- * @reg: Memory region allocated for the normal-mode suspend buffer.
+ * @gpu_va: The start GPU VA address of the bound suspend buffer. Note, this
+ * field is only valid when the owner group has a region bound at
+ * runtime.
* @phy: Array of physical memory pages allocated for the normal-
* mode suspend buffer.
*/
struct kbase_normal_suspend_buffer {
- struct kbase_va_region *reg;
+ u64 gpu_va;
struct tagged_addr *phy;
};
/**
* struct kbase_protected_suspend_buffer - Object representing a protected
* suspend buffer for queue group.
- * @reg: Memory region allocated for the protected-mode suspend buffer.
+ * @gpu_va: The start GPU VA address of the bound protected mode suspend buffer.
+ * Note, this field is only valid when the owner group has a region
+ * bound at runtime.
* @pma: Array of pointer to protected mode allocations containing
* information about memory pages allocated for protected mode
* suspend buffer.
+ * @alloc_retries: Number of times we retried allocing physical pages
+ * for protected suspend buffers.
*/
struct kbase_protected_suspend_buffer {
- struct kbase_va_region *reg;
+ u64 gpu_va;
struct protected_memory_allocation **pma;
+ u8 alloc_retries;
};
/**
@@ -406,6 +477,7 @@
* allowed to use.
* @compute_max: Maximum number of compute endpoints the group is
* allowed to use.
+ * @csi_handlers: Requested CSI exception handler flags for the group.
* @tiler_mask: Mask of tiler endpoints the group is allowed to use.
* @fragment_mask: Mask of fragment endpoints the group is allowed to use.
* @compute_mask: Mask of compute endpoints the group is allowed to use.
@@ -427,6 +499,12 @@
* @faulted: Indicates that a GPU fault occurred for the queue group.
* This flag persists until the fault has been queued to be
* reported to userspace.
+ * @cs_unrecoverable: Flag to unblock the thread waiting for CSG termination in
+ * case of CS_FATAL_EXCEPTION_TYPE_CS_UNRECOVERABLE
+ * @reevaluate_idle_status : Flag set when work is submitted for the normal group
+ * or it becomes unblocked during protected mode. The
+ * flag helps Scheduler confirm if the group actually
+ * became non idle or not.
* @bound_queues: Array of registered queues bound to this queue group.
* @doorbell_nr: Index of the hardware doorbell page assigned to the
* group.
@@ -442,6 +520,16 @@
* to be returned to userspace if such an error has occurred.
* @timer_event_work: Work item to handle the progress timeout fatal event
* for the group.
+ * @deschedule_deferred_cnt: Counter keeping a track of the number of threads
+ * that tried to deschedule the group and had to defer
+ * the descheduling due to the dump on fault.
+ * @csg_reg: An opaque pointer to the runtime bound shared regions. It is
+ * dynamically managed by the scheduler and can be NULL if the
+ * group is off-slot.
+ * @csg_reg_bind_retries: Runtime MCU shared region map operation attempted counts.
+ * It is accumulated on consecutive mapping attempt failures. On
+ * reaching a preset limit, the group is regarded as suffered
+ * a fatal error and triggers a fatal error notification.
*/
struct kbase_queue_group {
struct kbase_context *kctx;
@@ -454,6 +542,7 @@
u8 tiler_max;
u8 fragment_max;
u8 compute_max;
+ u8 csi_handlers;
u64 tiler_mask;
u64 fragment_mask;
@@ -467,6 +556,8 @@
u32 prepared_seq_num;
u32 scan_seq_num;
bool faulted;
+ bool cs_unrecoverable;
+ bool reevaluate_idle_status;
struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP];
@@ -479,6 +570,18 @@
struct kbase_csf_notification error_tiler_oom;
struct work_struct timer_event_work;
+
+ /**
+ * @dvs_buf: Address and size of scratch memory.
+ *
+ * Used to store intermediate DVS data by the GPU.
+ */
+ u64 dvs_buf;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+ u32 deschedule_deferred_cnt;
+#endif
+ void *csg_reg;
+ u8 csg_reg_bind_retries;
};
/**
@@ -488,10 +591,10 @@
* @lock: Lock preventing concurrent access to @array and the @in_use bitmap.
* @array: Array of pointers to kernel CPU command queues.
* @in_use: Bitmap which indicates which kernel CPU command queues are in use.
- * @wq: Dedicated workqueue for processing kernel CPU command queues.
- * @num_cmds: The number of commands that have been enqueued across
- * all the KCPU command queues. This could be used as a
- * timestamp to determine the command's enqueueing time.
+ * @cmd_seq_num: The sequence number assigned to an enqueued command,
+ * in incrementing order (older commands shall have a
+ * smaller number).
+ * @jit_lock: Lock to serialise JIT operations.
* @jit_cmds_head: A list of the just-in-time memory commands, both
* allocate & free, in submission order, protected
* by kbase_csf_kcpu_queue_context.lock.
@@ -504,9 +607,9 @@
struct mutex lock;
struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES];
DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES);
- struct workqueue_struct *wq;
- u64 num_cmds;
+ atomic64_t cmd_seq_num;
+ struct mutex jit_lock;
struct list_head jit_cmds_head;
struct list_head jit_blocked_queues;
};
@@ -530,10 +633,6 @@
/**
* struct kbase_csf_heap_context_allocator - Allocator of heap contexts
*
- * Heap context structures are allocated by the kernel for use by the firmware.
- * The current implementation subdivides a single GPU memory region for use as
- * a sparse array.
- *
* @kctx: Pointer to the kbase context with which this allocator is
* associated.
* @region: Pointer to a GPU memory region from which heap context structures
@@ -544,6 +643,12 @@
* @lock: Lock preventing concurrent access to the @in_use bitmap.
* @in_use: Bitmap that indicates which heap context structures are currently
* allocated (in @region).
+ * @heap_context_size_aligned: Size of a heap context structure, in bytes,
+ * aligned to GPU cacheline size.
+ *
+ * Heap context structures are allocated by the kernel for use by the firmware.
+ * The current implementation subdivides a single GPU memory region for use as
+ * a sparse array.
*/
struct kbase_csf_heap_context_allocator {
struct kbase_context *kctx;
@@ -551,27 +656,52 @@
u64 gpu_va;
struct mutex lock;
DECLARE_BITMAP(in_use, MAX_TILER_HEAPS);
+ u32 heap_context_size_aligned;
};
/**
* struct kbase_csf_tiler_heap_context - Object representing the tiler heaps
* context for a GPU address space.
*
- * This contains all of the CSF state relating to chunked tiler heaps for one
- * @kbase_context. It is not the same as a heap context structure allocated by
- * the kernel for use by the firmware.
- *
- * @lock: Lock preventing concurrent access to the tiler heaps.
+ * @lock: Lock to prevent the concurrent access to tiler heaps (after the
+ * initialization), a tiler heap can be terminated whilst an OoM
+ * event is being handled for it.
* @list: List of tiler heaps.
* @ctx_alloc: Allocator for heap context structures.
* @nr_of_heaps: Total number of tiler heaps that were added during the
* life time of the context.
+ *
+ * This contains all of the CSF state relating to chunked tiler heaps for one
+ * @kbase_context. It is not the same as a heap context structure allocated by
+ * the kernel for use by the firmware.
*/
struct kbase_csf_tiler_heap_context {
struct mutex lock;
struct list_head list;
struct kbase_csf_heap_context_allocator ctx_alloc;
u64 nr_of_heaps;
+};
+
+/**
+ * struct kbase_csf_ctx_heap_reclaim_info - Object representing the data section of
+ * a kctx for tiler heap reclaim manger
+ * @mgr_link: Link for hooking up to the heap reclaim manger's kctx lists
+ * @nr_freed_pages: Number of freed pages from the the kctx, after its attachment
+ * to the reclaim manager. This is used for tracking reclaim's
+ * free operation progress.
+ * @nr_est_unused_pages: Estimated number of pages that could be freed for the kctx
+ * when all its CSGs are off-slot, on attaching to the reclaim
+ * manager.
+ * @on_slot_grps: Number of on-slot groups from this kctx. In principle, if a
+ * kctx has groups on-slot, the scheduler will detach it from
+ * the tiler heap reclaim manager, i.e. no tiler heap memory
+ * reclaiming operations on the kctx.
+ */
+struct kbase_csf_ctx_heap_reclaim_info {
+ struct list_head mgr_link;
+ u32 nr_freed_pages;
+ u32 nr_est_unused_pages;
+ u8 on_slot_grps;
};
/**
@@ -595,6 +725,10 @@
* streams bound to groups of @idle_wait_groups list.
* @ngrp_to_schedule: Number of groups added for the context to the
* 'groups_to_schedule' list of scheduler instance.
+ * @heap_info: Heap reclaim information data of the kctx. As the
+ * reclaim action needs to be coordinated with the scheduler
+ * operations, any manipulations on the data needs holding
+ * the scheduler's mutex lock.
*/
struct kbase_csf_scheduler_context {
struct list_head runnable_groups[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
@@ -604,6 +738,61 @@
struct workqueue_struct *sync_update_wq;
struct work_struct sync_update_work;
u32 ngrp_to_schedule;
+ struct kbase_csf_ctx_heap_reclaim_info heap_info;
+};
+
+/**
+ * enum kbase_csf_event_callback_action - return type for CSF event callbacks.
+ *
+ * @KBASE_CSF_EVENT_CALLBACK_FIRST: Never set explicitly.
+ * It doesn't correspond to any action or type of event callback.
+ *
+ * @KBASE_CSF_EVENT_CALLBACK_KEEP: The callback will remain registered.
+ *
+ * @KBASE_CSF_EVENT_CALLBACK_REMOVE: The callback will be removed
+ * immediately upon return.
+ *
+ * @KBASE_CSF_EVENT_CALLBACK_LAST: Never set explicitly.
+ * It doesn't correspond to any action or type of event callback.
+ */
+enum kbase_csf_event_callback_action {
+ KBASE_CSF_EVENT_CALLBACK_FIRST = 0,
+ KBASE_CSF_EVENT_CALLBACK_KEEP,
+ KBASE_CSF_EVENT_CALLBACK_REMOVE,
+ KBASE_CSF_EVENT_CALLBACK_LAST,
+};
+
+/**
+ * struct kbase_csf_event - Object representing CSF event and error
+ *
+ * @callback_list: List of callbacks which are registered to serve CSF
+ * events.
+ * @error_list: List for CS fatal errors in CSF context.
+ * Link of fatal error is &struct_kbase_csf_notification.link.
+ * @lock: Lock protecting access to @callback_list and
+ * @error_list.
+ */
+struct kbase_csf_event {
+ struct list_head callback_list;
+ struct list_head error_list;
+ spinlock_t lock;
+};
+
+/**
+ * struct kbase_csf_user_reg_context - Object containing members to manage the mapping
+ * of USER Register page for a context.
+ *
+ * @vma: Pointer to the VMA corresponding to the virtual mapping
+ * of the USER register page.
+ * @file_offset: File offset value that is assigned to userspace mapping
+ * of the USER Register page. It is in page units.
+ * @link: Links the context to the device list when mapping is pointing to
+ * either the dummy or the real Register page.
+ */
+struct kbase_csf_user_reg_context {
+ struct vm_area_struct *vma;
+ u32 file_offset;
+ struct list_head link;
};
/**
@@ -637,26 +826,18 @@
* userspace mapping created for them on bind operation
* hasn't been removed.
* @kcpu_queues: Kernel CPU command queues.
- * @event_lock: Lock protecting access to @event_callback_list and
- * @error_list.
- * @event_callback_list: List of callbacks which are registered to serve CSF
- * events.
+ * @event: CSF event object.
* @tiler_heaps: Chunked tiler memory heaps.
* @wq: Dedicated workqueue to process work items corresponding
* to the OoM events raised for chunked tiler heaps being
* used by GPU command queues, and progress timeout events.
* @link: Link to this csf context in the 'runnable_kctxs' list of
* the scheduler instance
- * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping
- * of the USER register page. Currently used only for sanity
- * checking.
* @sched: Object representing the scheduler's context
- * @error_list: List for CS fatal errors in this context.
- * Link of fatal error is
- * &struct_kbase_csf_notification.link.
- * @event_lock needs to be held to access this list.
+ * @pending_submission_work: Work item to process pending kicked GPU command queues.
* @cpu_queue: CPU queue information. Only be available when DEBUG_FS
* is enabled.
+ * @user_reg: Collective information to support mapping to USER Register page.
*/
struct kbase_csf_context {
struct list_head event_pages_head;
@@ -667,17 +848,16 @@
struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM];
struct list_head queue_list;
struct kbase_csf_kcpu_queue_context kcpu_queues;
- spinlock_t event_lock;
- struct list_head event_callback_list;
+ struct kbase_csf_event event;
struct kbase_csf_tiler_heap_context tiler_heaps;
struct workqueue_struct *wq;
struct list_head link;
- struct vm_area_struct *user_reg_vma;
struct kbase_csf_scheduler_context sched;
- struct list_head error_list;
+ struct work_struct pending_submission_work;
#if IS_ENABLED(CONFIG_DEBUG_FS)
struct kbase_csf_cpu_queue_context cpu_queue;
#endif
+ struct kbase_csf_user_reg_context user_reg;
};
/**
@@ -714,6 +894,49 @@
atomic_t state;
unsigned long trigger_jiffies;
u8 priority;
+};
+
+/**
+ * struct kbase_csf_sched_heap_reclaim_mgr - Object for managing tiler heap reclaim
+ * kctx lists inside the CSF device's scheduler.
+ *
+ * @heap_reclaim: Tiler heap reclaim shrinker object.
+ * @ctx_lists: Array of kctx lists, size matching CSG defined priorities. The
+ * lists track the kctxs attached to the reclaim manager.
+ * @unused_pages: Estimated number of unused pages from the @ctxlist array. The
+ * number is indicative for use with reclaim shrinker's count method.
+ */
+struct kbase_csf_sched_heap_reclaim_mgr {
+ struct shrinker heap_reclaim;
+ struct list_head ctx_lists[KBASE_QUEUE_GROUP_PRIORITY_COUNT];
+ atomic_t unused_pages;
+};
+
+/**
+ * struct kbase_csf_mcu_shared_regions - Control data for managing the MCU shared
+ * interface segment regions for scheduler
+ * operations
+ *
+ * @array_csg_regs: Base pointer of an internally created array_csg_regs[].
+ * @unused_csg_regs: List contains unused csg_regs items. When an item is bound to a
+ * group that is placed onto on-slot by the scheduler, it is dropped
+ * from the list (i.e busy active). The Scheduler will put an active
+ * item back when it's becoming off-slot (not in use).
+ * @dummy_phys: An array of dummy phys[nr_susp_pages] pages for use with normal
+ * and pmode suspend buffers, as a default replacement of a CSG's pages
+ * for the MMU mapping when the csg_reg is not bound to a group.
+ * @pma_phys: Pre-allocated array phy[nr_susp_pages] for transitional use with
+ * protected suspend buffer MMU map operations.
+ * @userio_mem_rd_flags: Userio input page's read access mapping configuration flags.
+ * @dummy_phys_allocated: Indicating the @p dummy_phy page is allocated when true.
+ */
+struct kbase_csf_mcu_shared_regions {
+ void *array_csg_regs;
+ struct list_head unused_csg_regs;
+ struct tagged_addr *dummy_phys;
+ struct tagged_addr *pma_phys;
+ unsigned long userio_mem_rd_flags;
+ bool dummy_phys_allocated;
};
/**
@@ -788,6 +1011,8 @@
* operation to implement timeslice-based scheduling.
* @tock_work: Work item that would perform the schedule on tock
* operation to implement the asynchronous scheduling.
+ * @pending_tock_work: Indicates that the tock work item should re-execute
+ * once it's finished instead of going back to sleep.
* @ping_work: Work item that would ping the firmware at regular
* intervals, only if there is a single active CSG
* slot, to check if firmware is alive and would
@@ -797,16 +1022,29 @@
* @top_grp.
* @top_grp: Pointer to queue group inside @groups_to_schedule
* list that was assigned the highest slot priority.
- * @tock_pending_request: A "tock" request is pending: a group that is not
- * currently on the GPU demands to be scheduled.
* @active_protm_grp: Indicates if firmware has been permitted to let GPU
* enter protected mode with the given group. On exit
* from protected mode the pointer is reset to NULL.
- * @gpu_idle_fw_timer_enabled: Whether the CSF scheduler has activiated the
- * firmware idle hysteresis timer for preparing a
- * GPU suspend on idle.
+ * This pointer is set and PROTM_ENTER request is sent
+ * atomically with @interrupt_lock held.
+ * This pointer being set doesn't necessarily indicates
+ * that GPU is in protected mode, kbdev->protected_mode
+ * needs to be checked for that.
+ * @idle_wq: Workqueue for executing GPU idle notification
+ * handler.
* @gpu_idle_work: Work item for facilitating the scheduler to bring
* the GPU to a low-power mode on becoming idle.
+ * @fast_gpu_idle_handling: Indicates whether to relax many of the checks
+ * normally done in the GPU idle worker. This is
+ * set to true when handling the GLB IDLE IRQ if the
+ * idle hysteresis timeout is 0, since it makes it
+ * possible to receive this IRQ before the extract
+ * offset is published (which would cause more
+ * extensive GPU idle checks to fail).
+ * @gpu_no_longer_idle: Effective only when the GPU idle worker has been
+ * queued for execution, this indicates whether the
+ * GPU has become non-idle since the last time the
+ * idle notification was received.
* @non_idle_offslot_grps: Count of off-slot non-idle groups. Reset during
* the scheduler active phase in a tick. It then
* tracks the count of non-idle groups across all the
@@ -827,6 +1065,17 @@
* when scheduling tick needs to be advanced from
* interrupt context, without actually deactivating
* the @tick_timer first and then enqueing @tick_work.
+ * @tick_protm_pending_seq: Scan out sequence number of the group that has
+ * protected mode execution pending for the queue(s)
+ * bound to it and will be considered first for the
+ * protected mode execution compared to other such
+ * groups. It is updated on every tick/tock.
+ * @interrupt_lock is used to serialize the access.
+ * @protm_enter_time: GPU protected mode enter time.
+ * @reclaim_mgr: CSGs tiler heap manager object.
+ * @mcu_regs_data: Scheduler MCU shared regions data for managing the
+ * shared interface mappings for on-slot queues and
+ * CSG suspend buffers.
*/
struct kbase_csf_scheduler {
struct mutex lock;
@@ -853,26 +1102,32 @@
struct hrtimer tick_timer;
struct work_struct tick_work;
struct delayed_work tock_work;
+ atomic_t pending_tock_work;
struct delayed_work ping_work;
struct kbase_context *top_ctx;
struct kbase_queue_group *top_grp;
- bool tock_pending_request;
struct kbase_queue_group *active_protm_grp;
- bool gpu_idle_fw_timer_enabled;
+ struct workqueue_struct *idle_wq;
struct work_struct gpu_idle_work;
+ bool fast_gpu_idle_handling;
+ atomic_t gpu_no_longer_idle;
atomic_t non_idle_offslot_grps;
u32 non_idle_scanout_grps;
u32 pm_active_count;
unsigned int csg_scheduling_period_ms;
bool tick_timer_active;
+ u32 tick_protm_pending_seq;
+ ktime_t protm_enter_time;
+ struct kbase_csf_sched_heap_reclaim_mgr reclaim_mgr;
+ struct kbase_csf_mcu_shared_regions mcu_regs_data;
};
-/**
+/*
* Number of GPU cycles per unit of the global progress timeout.
*/
#define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024)
-/**
+/*
* Maximum value of the global progress timeout.
*/
#define GLB_PROGRESS_TIMER_TIMEOUT_MAX \
@@ -880,12 +1135,12 @@
GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * \
GLB_PROGRESS_TIMER_TIMEOUT_SCALE)
-/**
+/*
* Default GLB_PWROFF_TIMER_TIMEOUT value in unit of micro-seconds.
*/
#define DEFAULT_GLB_PWROFF_TIMEOUT_US (800)
-/**
+/*
* In typical operations, the management of the shader core power transitions
* is delegated to the MCU/firmware. However, if the host driver is configured
* to take direct control, one needs to disable the MCU firmware GLB_PWROFF
@@ -896,7 +1151,7 @@
/* Index of the GPU_ACTIVE counter within the CSHW counter block */
#define GPU_ACTIVE_CNT_IDX (4)
-/**
+/*
* Maximum number of sessions that can be managed by the IPA Control component.
*/
#if MALI_UNIT_TEST
@@ -922,13 +1177,13 @@
KBASE_IPA_CORE_TYPE_NUM
};
-/**
+/*
* Number of configurable counters per type of block on the IPA Control
* interface.
*/
#define KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS ((size_t)8)
-/**
+/*
* Total number of configurable counters existing on the IPA Control interface.
*/
#define KBASE_IPA_CONTROL_MAX_COUNTERS \
@@ -1001,8 +1256,7 @@
*
*/
struct kbase_ipa_control_prfcnt_block {
- struct kbase_ipa_control_prfcnt_config
- select[KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS];
+ struct kbase_ipa_control_prfcnt_config select[KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS];
size_t num_available_counters;
};
@@ -1025,8 +1279,7 @@
*/
struct kbase_ipa_control {
struct kbase_ipa_control_prfcnt_block blocks[KBASE_IPA_CORE_TYPE_NUM];
- struct kbase_ipa_control_session
- sessions[KBASE_IPA_CONTROL_MAX_SESSIONS];
+ struct kbase_ipa_control_session sessions[KBASE_IPA_CONTROL_MAX_SESSIONS];
spinlock_t lock;
void *rtm_listener_data;
size_t num_active_sessions;
@@ -1040,12 +1293,20 @@
* @node: Interface objects are on the kbase_device:csf.firmware_interfaces
* list using this list_head to link them
* @phys: Array of the physical (tagged) addresses making up this interface
+ * @reuse_pages: Flag used to identify if the FW interface entry reuses
+ * physical pages allocated for another FW interface entry.
+ * @is_small_page: Flag used to identify if small pages are used for
+ * the FW interface entry.
* @name: NULL-terminated string naming the interface
* @num_pages: Number of entries in @phys and @pma (and length of the interface)
+ * @num_pages_aligned: Same as @num_pages except for the case when @is_small_page
+ * is false and @reuse_pages is false and therefore will be
+ * aligned to NUM_4K_PAGES_IN_2MB_PAGE.
* @virtual: Starting GPU virtual address this interface is mapped at
* @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes
* @data_start: Offset into firmware image at which the interface data starts
* @data_end: Offset into firmware image at which the interface data ends
+ * @virtual_exe_start: Starting GPU execution virtual address of this interface
* @kernel_map: A kernel mapping of the memory or NULL if not required to be
* mapped in the kernel
* @pma: Array of pointers to protected memory allocations.
@@ -1053,12 +1314,16 @@
struct kbase_csf_firmware_interface {
struct list_head node;
struct tagged_addr *phys;
+ bool reuse_pages;
+ bool is_small_page;
char *name;
u32 num_pages;
+ u32 num_pages_aligned;
u32 virtual;
u32 flags;
u32 data_start;
u32 data_end;
+ u32 virtual_exe_start;
void *kernel_map;
struct protected_memory_allocation **pma;
};
@@ -1075,6 +1340,136 @@
struct kbase_csf_hwcnt {
bool request_pending;
bool enable_pending;
+};
+
+/*
+ * struct kbase_csf_mcu_fw - Object containing device loaded MCU firmware data.
+ *
+ * @size: Loaded firmware data size. Meaningful only when the
+ * other field @p data is not NULL.
+ * @data: Pointer to the device retained firmware data. If NULL
+ * means not loaded yet or error in loading stage.
+ */
+struct kbase_csf_mcu_fw {
+ size_t size;
+ u8 *data;
+};
+
+/*
+ * Firmware log polling period.
+ */
+#define KBASE_CSF_FIRMWARE_LOG_POLL_PERIOD_MS 25
+
+/**
+ * enum kbase_csf_firmware_log_mode - Firmware log operating mode
+ *
+ * @KBASE_CSF_FIRMWARE_LOG_MODE_MANUAL: Manual mode, firmware log can be read
+ * manually by the userspace (and it will also be dumped automatically into
+ * dmesg on GPU reset).
+ *
+ * @KBASE_CSF_FIRMWARE_LOG_MODE_AUTO_PRINT: Automatic printing mode, firmware log
+ * will be periodically emptied into dmesg, manual reading through debugfs is
+ * disabled.
+ */
+enum kbase_csf_firmware_log_mode {
+ KBASE_CSF_FIRMWARE_LOG_MODE_MANUAL,
+ KBASE_CSF_FIRMWARE_LOG_MODE_AUTO_PRINT
+};
+
+/**
+ * struct kbase_csf_firmware_log - Object containing members for handling firmware log.
+ *
+ * @mode: Firmware log operating mode.
+ * @busy: Indicating whether a firmware log operation is in progress.
+ * @poll_work: Work item that would poll firmware log buffer
+ * at regular intervals to perform any periodic
+ * activities required by current log mode.
+ * @dump_buf: Buffer used for dumping the log.
+ * @func_call_list_va_start: Virtual address of the start of the call list of FW log functions.
+ * @func_call_list_va_end: Virtual address of the end of the call list of FW log functions.
+ */
+struct kbase_csf_firmware_log {
+ enum kbase_csf_firmware_log_mode mode;
+ atomic_t busy;
+ struct delayed_work poll_work;
+ u8 *dump_buf;
+ u32 func_call_list_va_start;
+ u32 func_call_list_va_end;
+};
+
+/**
+ * struct kbase_csf_firmware_core_dump - Object containing members for handling
+ * firmware core dump.
+ *
+ * @mcu_regs_addr: GPU virtual address of the start of the MCU registers buffer
+ * in Firmware.
+ * @version: Version of the FW image header core dump data format. Bits
+ * 7:0 specify version minor and 15:8 specify version major.
+ * @available: Flag to identify if the FW core dump buffer is available.
+ * True if entry is available in the FW image header and version
+ * is supported, False otherwise.
+ */
+struct kbase_csf_firmware_core_dump {
+ u32 mcu_regs_addr;
+ u16 version;
+ bool available;
+};
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+/**
+ * struct kbase_csf_dump_on_fault - Faulty information to deliver to the daemon
+ *
+ * @error_code: Error code.
+ * @kctx_tgid: tgid value of the Kbase context for which the fault happened.
+ * @kctx_id: id of the Kbase context for which the fault happened.
+ * @enabled: Flag to indicate that 'csf_fault' debugfs has been opened
+ * so dump on fault is enabled.
+ * @fault_wait_wq: Waitqueue on which user space client is blocked till kbase
+ * reports a fault.
+ * @dump_wait_wq: Waitqueue on which kbase threads are blocked till user space client
+ * completes the dump on fault.
+ * @lock: Lock to protect this struct members from concurrent access.
+ */
+struct kbase_csf_dump_on_fault {
+ enum dumpfault_error_type error_code;
+ u32 kctx_tgid;
+ u32 kctx_id;
+ atomic_t enabled;
+ wait_queue_head_t fault_wait_wq;
+ wait_queue_head_t dump_wait_wq;
+ spinlock_t lock;
+};
+#endif /* CONFIG_DEBUG_FS*/
+
+/**
+ * struct kbase_csf_user_reg - Object containing members to manage the mapping
+ * of USER Register page for all contexts
+ *
+ * @dummy_page: Address of a dummy page that is mapped in place
+ * of the real USER Register page just before the GPU
+ * is powered down. The USER Register page is mapped
+ * in the address space of every process, that created
+ * a Base context, to enable the access to LATEST_FLUSH
+ * register from userspace.
+ * @filp: Pointer to a dummy file, that along with @file_offset,
+ * facilitates the use of unique file offset for the userspace mapping
+ * created for USER Register page.
+ * The userspace mapping is made to point to this file
+ * inside the mmap handler.
+ * @file_offset: Counter that is incremented every time Userspace creates a mapping of
+ * USER Register page, to provide a unique file offset range for
+ * @filp file, so that the CPU PTE of the Userspace mapping can be zapped
+ * through the kernel function unmap_mapping_range().
+ * It is incremented in page units.
+ * @list: Linked list to maintain user processes(contexts)
+ * having the mapping to USER Register page.
+ * It's protected by &kbase_csf_device.reg_lock.
+ */
+struct kbase_csf_user_reg {
+ struct tagged_addr dummy_page;
+ struct file *filp;
+ u32 file_offset;
+ struct list_head list;
};
/**
@@ -1114,21 +1509,10 @@
* of the real Hw doorbell page for the active GPU
* command queues after they are stopped or after the
* GPU is powered down.
- * @dummy_user_reg_page: Address of the dummy page that is mapped in place
- * of the real User register page just before the GPU
- * is powered down. The User register page is mapped
- * in the address space of every process, that created
- * a Base context, to enable the access to LATEST_FLUSH
- * register from userspace.
- * @mali_file_inode: Pointer to the inode corresponding to mali device
- * file. This is needed in order to switch to the
- * @dummy_user_reg_page on GPU power down.
- * All instances of the mali device file will point to
- * the same inode.
* @reg_lock: Lock to serialize the MCU firmware related actions
* that affect all contexts such as allocation of
* regions from shared interface area, assignment of
- * of hardware doorbell pages, assignment of CSGs,
+ * hardware doorbell pages, assignment of CSGs,
* sending global requests.
* @event_wait: Wait queue to wait for receiving csf events, i.e.
* the interrupt from CSF firmware, or scheduler state
@@ -1151,6 +1535,10 @@
* in GPU reset has completed.
* @firmware_reload_needed: Flag for indicating that the firmware needs to be
* reloaded as part of the GPU reset action.
+ * @firmware_full_reload_needed: Flag for indicating that the firmware needs to
+ * be fully re-loaded. This may be set when the
+ * boot or re-init of MCU fails after a successful
+ * soft reset.
* @firmware_hctl_core_pwr: Flag for indicating that the host diver is in
* charge of the shader core's power transitions, and
* the mcu_core_pwroff timeout feature is disabled
@@ -1174,9 +1562,9 @@
* the glb_pwoff register. This is separated from
* the @p mcu_core_pwroff_dur_count as an update
* to the latter is asynchronous.
- * @gpu_idle_hysteresis_ms: Sysfs attribute for the idle hysteresis time
- * window in unit of ms. The firmware does not use it
- * directly.
+ * @gpu_idle_hysteresis_us: Sysfs attribute for the idle hysteresis time
+ * window in unit of microseconds. The firmware does not
+ * use it directly.
* @gpu_idle_dur_count: The counterpart of the hysteresis time window in
* interface required format, ready to be used
* directly in the firmware.
@@ -1184,6 +1572,13 @@
* for any request sent to the firmware.
* @hwcnt: Contain members required for handling the dump of
* HW counters.
+ * @fw: Copy of the loaded MCU firmware image.
+ * @fw_log: Contain members required for handling firmware log.
+ * @fw_core_dump: Contain members required for handling the firmware
+ * core dump.
+ * @dof: Structure for dump on fault.
+ * @user_reg: Collective information to support the mapping to
+ * USER Register page for user processes.
*/
struct kbase_csf_device {
struct kbase_mmu_table mcu_mmu;
@@ -1197,8 +1592,6 @@
struct file *db_filp;
u32 db_file_offsets;
struct tagged_addr dummy_db_page;
- struct tagged_addr dummy_user_reg_page;
- struct inode *mali_file_inode;
struct mutex reg_lock;
wait_queue_head_t event_wait;
bool interrupt_received;
@@ -1210,6 +1603,7 @@
bool firmware_inited;
bool firmware_reloaded;
bool firmware_reload_needed;
+ bool firmware_full_reload_needed;
bool firmware_hctl_core_pwr;
struct work_struct firmware_reload_work;
bool glb_init_request_pending;
@@ -1218,10 +1612,23 @@
u32 mcu_core_pwroff_dur_us;
u32 mcu_core_pwroff_dur_count;
u32 mcu_core_pwroff_reg_shadow;
- u32 gpu_idle_hysteresis_ms;
+ u32 gpu_idle_hysteresis_us;
u32 gpu_idle_dur_count;
unsigned int fw_timeout_ms;
struct kbase_csf_hwcnt hwcnt;
+ struct kbase_csf_mcu_fw fw;
+ struct kbase_csf_firmware_log fw_log;
+ struct kbase_csf_firmware_core_dump fw_core_dump;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+ struct kbase_csf_dump_on_fault dof;
+#endif /* CONFIG_DEBUG_FS */
+#if IS_ENABLED(CONFIG_MALI_CORESIGHT)
+ /**
+ * @coresight: Coresight device structure.
+ */
+ struct kbase_debug_coresight_device coresight;
+#endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */
+ struct kbase_csf_user_reg user_reg;
};
/**
@@ -1238,6 +1645,10 @@
* @bf_data: Data relating to Bus fault.
* @gf_data: Data relating to GPU fault.
* @current_setup: Stores the MMU configuration for this address space.
+ * @is_unresponsive: Flag to indicate MMU is not responding.
+ * Set if a MMU command isn't completed within
+ * &kbase_device:mmu_as_inactive_wait_time_ms.
+ * Clear by kbase_ctx_sched_restore_all_as() after GPU reset completes.
*/
struct kbase_as {
int number;
@@ -1249,6 +1660,7 @@
struct kbase_fault bf_data;
struct kbase_fault gf_data;
struct kbase_mmu_setup current_setup;
+ bool is_unresponsive;
};
#endif /* _KBASE_CSF_DEFS_H_ */
--
Gitblit v1.6.2