From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c | 374 ++++++++++++++++++++++++-----------------------------
1 files changed, 168 insertions(+), 206 deletions(-)
diff --git a/kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
index e7e9fb1..c725711 100644
--- a/kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
+++ b/kernel/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -33,7 +33,8 @@
#include <mali_kbase_reset_gpu.h>
#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_kinstr_jm.h>
-#include <mali_kbase_hwcnt_context.h>
+#include <mali_kbase_hwaccess_instr.h>
+#include <hwcnt/mali_kbase_hwcnt_context.h>
#include <device/mali_kbase_device.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
@@ -43,23 +44,17 @@
static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev,
const u64 affinity, const u64 limited_core_mask);
-static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
- base_jd_core_req core_req,
- int js, const u64 limited_core_mask)
+static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req core_req,
+ unsigned int js, const u64 limited_core_mask)
{
u64 affinity;
+ bool skip_affinity_check = false;
if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) ==
BASE_JD_REQ_T) {
- /* Tiler-only atom */
- /* If the hardware supports XAFFINITY then we'll only enable
- * the tiler (which is the default so this is a no-op),
- * otherwise enable shader core 0.
- */
- if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY))
- affinity = 1;
- else
- affinity = 0;
+ /* Tiler-only atom, affinity value can be programed as 0 */
+ affinity = 0;
+ skip_affinity_check = true;
} else if ((core_req & (BASE_JD_REQ_COHERENT_GROUP |
BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) {
unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
@@ -89,7 +84,7 @@
affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask);
}
- if (unlikely(!affinity)) {
+ if (unlikely(!affinity && !skip_affinity_check)) {
#ifdef CONFIG_MALI_BIFROST_DEBUG
u64 shaders_ready =
kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
@@ -195,22 +190,42 @@
return jc;
}
-void kbase_job_hw_submit(struct kbase_device *kbdev,
- struct kbase_jd_atom *katom,
- int js)
+static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js,
+ struct kbase_context *kctx)
+{
+ const ktime_t wait_loop_start = ktime_get_raw();
+ const s64 max_timeout = (s64)kbdev->js_data.js_free_wait_time_ms;
+ s64 diff = 0;
+
+ /* wait for the JS_COMMAND_NEXT register to reach the given status value */
+ do {
+ if (!kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)))
+ return true;
+
+ diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start));
+ } while (diff < max_timeout);
+
+ dev_err(kbdev->dev, "Timeout in waiting for job slot %u to become free for ctx %d_%u", js,
+ kctx->tgid, kctx->id);
+
+ return false;
+}
+
+int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js)
{
struct kbase_context *kctx;
u32 cfg;
u64 const jc_head = select_job_chain(katom);
u64 affinity;
+ struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js];
- KBASE_DEBUG_ASSERT(kbdev);
- KBASE_DEBUG_ASSERT(katom);
+ lockdep_assert_held(&kbdev->hwaccess_lock);
kctx = katom->kctx;
/* Command register must be available */
- KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
+ if (!kbasep_jm_wait_js_free(kbdev, js, kctx))
+ return -EPERM;
dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %pK\n",
jc_head, (void *)katom);
@@ -232,9 +247,23 @@
!(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET))
cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
- if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START))
- cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
- else
+ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) {
+ /* Force a cache maintenance operation if the newly submitted
+ * katom to the slot is from a different kctx. For a JM GPU
+ * that has the feature BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER,
+ * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a
+ * FLUSH_CLEAN_INVALIDATE.
+ */
+ u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged;
+
+ if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) {
+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER))
+ cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER;
+ else
+ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
+ } else
+ cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
+ } else
cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) &&
@@ -251,18 +280,13 @@
(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
- if (kbase_hw_has_feature(kbdev,
- BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
- if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
- cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
- katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
- kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
- true;
- } else {
- katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
- kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
- false;
- }
+ if (!ptr_slot_rb->job_chain_flag) {
+ cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
+ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
+ ptr_slot_rb->job_chain_flag = true;
+ } else {
+ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
+ ptr_slot_rb->job_chain_flag = false;
}
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg);
@@ -274,7 +298,7 @@
/* Write an approximate start timestamp.
* It's approximate because there might be a job in the HEAD register.
*/
- katom->start_timestamp = ktime_get();
+ katom->start_timestamp = ktime_get_raw();
/* GO ! */
dev_dbg(kbdev->dev, "JS: Submitting atom %pK from ctx %pK to js[%d] with head=0x%llx",
@@ -300,6 +324,10 @@
&kbdev->gpu_props.props.raw_props.js_features[js],
"ctx_nr,atom_nr");
kbase_kinstr_jm_atom_hw_submit(katom);
+
+ /* Update the slot's last katom submission kctx */
+ ptr_slot_rb->last_kctx_tagged = SLOT_RB_TAG_KCTX(kctx);
+
#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS)
if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
/* If this is the only job on the slot, trace it as starting */
@@ -310,7 +338,6 @@
sizeof(js_string)),
ktime_to_ns(katom->start_timestamp),
(u32)katom->kctx->id, 0, katom->work_id);
- kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
}
#endif
@@ -319,6 +346,8 @@
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
JS_COMMAND_START);
+
+ return 0;
}
/**
@@ -334,10 +363,8 @@
* work out the best estimate (which might still result in an over-estimate to
* the calculated time spent)
*/
-static void kbasep_job_slot_update_head_start_timestamp(
- struct kbase_device *kbdev,
- int js,
- ktime_t end_timestamp)
+static void kbasep_job_slot_update_head_start_timestamp(struct kbase_device *kbdev, unsigned int js,
+ ktime_t end_timestamp)
{
ktime_t timestamp_diff;
struct kbase_jd_atom *katom;
@@ -367,8 +394,7 @@
* Make a tracepoint call to the instrumentation module informing that
* softstop happened on given lpu (job slot).
*/
-static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev,
- int js)
+static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, unsigned int js)
{
KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(
kbdev,
@@ -377,19 +403,17 @@
void kbase_job_done(struct kbase_device *kbdev, u32 done)
{
- int i;
u32 count = 0;
ktime_t end_timestamp;
lockdep_assert_held(&kbdev->hwaccess_lock);
- KBASE_DEBUG_ASSERT(kbdev);
-
KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done);
- end_timestamp = ktime_get();
+ end_timestamp = ktime_get_raw();
while (done) {
+ unsigned int i;
u32 failed = done >> 16;
/* treat failed slots as finished slots */
@@ -399,7 +423,6 @@
* numbered interrupts before the higher numbered ones.
*/
i = ffs(finished) - 1;
- KBASE_DEBUG_ASSERT(i >= 0);
do {
int nr_done;
@@ -415,6 +438,8 @@
JOB_SLOT_REG(i, JS_STATUS));
if (completion_code == BASE_JD_EVENT_STOPPED) {
+ u64 job_head;
+
KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(
kbdev, NULL,
i, 0, TL_JS_EVENT_SOFT_STOP);
@@ -431,6 +456,21 @@
((u64)kbase_reg_read(kbdev,
JOB_SLOT_REG(i, JS_TAIL_HI))
<< 32);
+ job_head = (u64)kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i, JS_HEAD_LO)) |
+ ((u64)kbase_reg_read(kbdev,
+ JOB_SLOT_REG(i, JS_HEAD_HI))
+ << 32);
+ /* For a soft-stopped job chain js_tail should
+ * same as the js_head, but if not then the
+ * job chain was incorrectly marked as
+ * soft-stopped. In such case we should not
+ * be resuming the job chain from js_tail and
+ * report the completion_code as UNKNOWN.
+ */
+ if (job_tail != job_head)
+ completion_code = BASE_JD_EVENT_UNKNOWN;
+
} else if (completion_code ==
BASE_JD_EVENT_NOT_STARTED) {
/* PRLAM-10673 can cause a TERMINATED
@@ -563,7 +603,7 @@
failed = done >> 16;
finished = (done & 0xFFFF) | failed;
if (done)
- end_timestamp = ktime_get();
+ end_timestamp = ktime_get_raw();
} while (finished & (1 << i));
kbasep_job_slot_update_head_start_timestamp(kbdev, i,
@@ -581,18 +621,16 @@
KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
}
-void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
- int js,
- u32 action,
- base_jd_core_req core_reqs,
- struct kbase_jd_atom *target_katom)
+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js,
+ u32 action, base_jd_core_req core_reqs,
+ struct kbase_jd_atom *target_katom)
{
#if KBASE_KTRACE_ENABLE
u32 status_reg_before;
u64 job_in_head_before;
u32 status_reg_after;
- KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
+ WARN_ON(action & (~JS_COMMAND_MASK));
/* Check the head pointer */
job_in_head_before = ((u64) kbase_reg_read(kbdev,
@@ -621,25 +659,17 @@
/* Mark the point where we issue the soft-stop command */
KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom);
- if (kbase_hw_has_feature(
- kbdev,
- BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
- action = (target_katom->atom_flags &
- KBASE_KATOM_FLAGS_JOBCHAIN) ?
- JS_COMMAND_SOFT_STOP_1 :
- JS_COMMAND_SOFT_STOP_0;
- }
+ action = (target_katom->atom_flags &
+ KBASE_KATOM_FLAGS_JOBCHAIN) ?
+ JS_COMMAND_SOFT_STOP_1 :
+ JS_COMMAND_SOFT_STOP_0;
} else if (action == JS_COMMAND_HARD_STOP) {
target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
- if (kbase_hw_has_feature(
- kbdev,
- BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
- action = (target_katom->atom_flags &
- KBASE_KATOM_FLAGS_JOBCHAIN) ?
- JS_COMMAND_HARD_STOP_1 :
- JS_COMMAND_HARD_STOP_0;
- }
+ action = (target_katom->atom_flags &
+ KBASE_KATOM_FLAGS_JOBCHAIN) ?
+ JS_COMMAND_HARD_STOP_1 :
+ JS_COMMAND_HARD_STOP_0;
}
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action);
@@ -651,6 +681,10 @@
struct kbase_context *head_kctx;
head = kbase_gpu_inspect(kbdev, js, 0);
+ if (unlikely(!head)) {
+ dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js);
+ return;
+ }
head_kctx = head->kctx;
if (status_reg_before == BASE_JD_EVENT_ACTIVE)
@@ -678,7 +712,8 @@
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
break;
default:
- BUG();
+ WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action,
+ (void *)target_katom, (void *)target_katom->kctx);
break;
}
} else {
@@ -707,7 +742,8 @@
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
break;
default:
- BUG();
+ WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action,
+ (void *)target_katom, (void *)target_katom->kctx);
break;
}
}
@@ -717,7 +753,7 @@
void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx)
{
struct kbase_device *kbdev = kctx->kbdev;
- int i;
+ unsigned int i;
lockdep_assert_held(&kbdev->hwaccess_lock);
@@ -725,69 +761,33 @@
kbase_job_slot_hardstop(kctx, i, NULL);
}
-/**
- * kbase_is_existing_atom_submitted_later_than_ready
- * @ready: sequence number of the ready atom
- * @existing: sequence number of the existing atom
- *
- * Returns true if the existing atom has been submitted later than the
- * ready atom. It is used to understand if an atom that is ready has been
- * submitted earlier than the currently running atom, so that the currently
- * running atom should be preempted to allow the ready atom to run.
- */
-static inline bool kbase_is_existing_atom_submitted_later_than_ready(u64 ready, u64 existing)
-{
- /* No seq_nr set? */
- if (!ready || !existing)
- return false;
-
- /* Efficiently handle the unlikely case of wrapping.
- * The following code assumes that the delta between the sequence number
- * of the two atoms is less than INT64_MAX.
- * In the extremely unlikely case where the delta is higher, the comparison
- * defaults for no preemption.
- * The code also assumes that the conversion from unsigned to signed types
- * works because the signed integers are 2's complement.
- */
- return (s64)(ready - existing) < 0;
-}
-
void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
struct kbase_jd_atom *target_katom)
{
struct kbase_device *kbdev;
- int js = target_katom->slot_nr;
- int priority = target_katom->sched_priority;
- int seq_nr = target_katom->seq_nr;
+ unsigned int target_js = target_katom->slot_nr;
int i;
bool stop_sent = false;
- KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = kctx->kbdev;
- KBASE_DEBUG_ASSERT(kbdev != NULL);
lockdep_assert_held(&kbdev->hwaccess_lock);
- for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
- struct kbase_jd_atom *katom;
+ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, target_js); i++) {
+ struct kbase_jd_atom *slot_katom;
- katom = kbase_gpu_inspect(kbdev, js, i);
- if (!katom)
+ slot_katom = kbase_gpu_inspect(kbdev, target_js, i);
+ if (!slot_katom)
continue;
- if ((kbdev->js_ctx_scheduling_mode ==
- KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) &&
- (katom->kctx != kctx))
- continue;
-
- if ((katom->sched_priority > priority) ||
- (katom->kctx == kctx && kbase_is_existing_atom_submitted_later_than_ready(seq_nr, katom->seq_nr))) {
+ if (kbase_js_atom_runs_before(kbdev, target_katom, slot_katom,
+ KBASE_ATOM_ORDERING_FLAG_SEQNR)) {
if (!stop_sent)
KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED(
kbdev,
target_katom);
- kbase_job_slot_softstop(kbdev, js, katom);
+ kbase_job_slot_softstop(kbdev, target_js, slot_katom);
stop_sent = true;
}
}
@@ -875,7 +875,7 @@
if (timeout != 0)
goto exit;
- if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) {
+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) {
dev_err(kbdev->dev,
"Issuing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
ZAP_TIMEOUT);
@@ -943,46 +943,29 @@
*
* Where possible any job in the next register is evicted before the soft-stop.
*/
-void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
- struct kbase_jd_atom *target_katom, u32 sw_flags)
+void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js,
+ struct kbase_jd_atom *target_katom, u32 sw_flags)
{
dev_dbg(kbdev->dev, "Soft-stop atom %pK with flags 0x%x (s:%d)\n",
target_katom, sw_flags, js);
- KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
+ if (sw_flags & JS_COMMAND_MASK) {
+ WARN(true, "Atom %pK in kctx %pK received non-NOP flags %d\n", (void *)target_katom,
+ target_katom ? (void *)target_katom->kctx : NULL, sw_flags);
+ sw_flags &= ~((u32)JS_COMMAND_MASK);
+ }
kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
JS_COMMAND_SOFT_STOP | sw_flags);
}
-/**
- * kbase_job_slot_softstop - Soft-stop the specified job slot
- * @kbdev: The kbase device
- * @js: The job slot to soft-stop
- * @target_katom: The job that should be soft-stopped (or NULL for any job)
- * Context:
- * The job slot lock must be held when calling this function.
- * The job slot must not already be in the process of being soft-stopped.
- *
- * Where possible any job in the next register is evicted before the soft-stop.
- */
void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
struct kbase_jd_atom *target_katom)
{
kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
}
-/**
- * kbase_job_slot_hardstop - Hard-stop the specified job slot
- * @kctx: The kbase context that contains the job(s) that should
- * be hard-stopped
- * @js: The job slot to hard-stop
- * @target_katom: The job that should be hard-stopped (or NULL for all
- * jobs from the context)
- * Context:
- * The job slot lock must be held when calling this function.
- */
-void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
- struct kbase_jd_atom *target_katom)
+void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js,
+ struct kbase_jd_atom *target_katom)
{
struct kbase_device *kbdev = kctx->kbdev;
bool stopped;
@@ -990,28 +973,9 @@
stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
target_katom,
JS_COMMAND_HARD_STOP);
+ CSTD_UNUSED(stopped);
}
-/**
- * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
- * @kbdev: kbase device
- * @action: the event which has occurred
- * @core_reqs: core requirements of the atom
- * @target_katom: the atom which is being affected
- *
- * For a certain soft-stop action, work out whether to enter disjoint
- * state.
- *
- * This does not register multiple disjoint events if the atom has already
- * started a disjoint period
- *
- * @core_reqs can be supplied as 0 if the atom had not started on the hardware
- * (and so a 'real' soft/hard-stop was not required, but it still interrupted
- * flow, perhaps on another context)
- *
- * kbase_job_check_leave_disjoint() should be used to end the disjoint
- * state when the soft/hard-stop action is complete
- */
void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom)
{
@@ -1033,14 +997,6 @@
kbase_disjoint_state_up(kbdev);
}
-/**
- * kbase_job_check_enter_disjoint - potentially leave disjoint state
- * @kbdev: kbase device
- * @target_katom: atom which is finishing
- *
- * Work out whether to leave disjoint state when finishing an atom that was
- * originated by kbase_job_check_enter_disjoint().
- */
void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
struct kbase_jd_atom *target_katom)
{
@@ -1117,21 +1073,21 @@
{
unsigned long flags;
struct kbase_device *kbdev;
- ktime_t end_timestamp = ktime_get();
+ ktime_t end_timestamp = ktime_get_raw();
struct kbasep_js_device_data *js_devdata;
bool silent = false;
u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
- KBASE_DEBUG_ASSERT(data);
-
kbdev = container_of(data, struct kbase_device,
hwaccess.backend.reset_work);
- KBASE_DEBUG_ASSERT(kbdev);
js_devdata = &kbdev->js_data;
if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
KBASE_RESET_GPU_SILENT)
+ silent = true;
+
+ if (kbase_is_quick_reset_enabled(kbdev))
silent = true;
KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
@@ -1162,7 +1118,7 @@
return;
}
- KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false);
+ WARN(kbdev->irq_reset_flush, "%s: GPU reset already in flight\n", __func__);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
spin_lock(&kbdev->mmu_mask_change);
@@ -1203,7 +1159,8 @@
mutex_lock(&kbdev->pm.lock);
/* We hold the pm lock, so there ought to be a current policy */
- KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
+ if (unlikely(!kbdev->pm.backend.pm_current_policy))
+ dev_warn(kbdev->dev, "No power policy set!");
/* All slot have been soft-stopped and we've waited
* SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
@@ -1228,6 +1185,13 @@
kbase_backend_reset(kbdev, &end_timestamp);
kbase_pm_metrics_update(kbdev, NULL);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+
+ /* Tell hardware counters a reset is about to occur.
+ * If the instr backend is in an unrecoverable error state (e.g. due to
+ * HW being unresponsive), this will transition the backend out of
+ * it, on the assumption a reset will fix whatever problem there was.
+ */
+ kbase_instr_hwcnt_on_before_reset(kbdev);
/* Reset the GPU */
kbase_pm_init_hw(kbdev, 0);
@@ -1293,8 +1257,6 @@
struct kbase_device *kbdev = container_of(timer, struct kbase_device,
hwaccess.backend.reset_timer);
- KBASE_DEBUG_ASSERT(kbdev);
-
/* Reset still pending? */
if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
@@ -1312,10 +1274,8 @@
static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
{
- int i;
+ unsigned int i;
int pending_jobs = 0;
-
- KBASE_DEBUG_ASSERT(kbdev);
/* Count the number of jobs */
for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
@@ -1361,11 +1321,10 @@
* @kbdev: kbase device
* @flags: Bitfield indicating impact of reset (see flag defines)
*
- * This function just soft-stops all the slots to ensure that as many jobs as
+ * This function soft-stops all the slots to ensure that as many jobs as
* possible are saved.
*
- * Return:
- * The function returns a boolean which should be interpreted as follows:
+ * Return: boolean which should be interpreted as follows:
* true - Prepared for reset, kbase_reset_gpu_locked should be called.
* false - Another thread is performing a reset, kbase_reset_gpu should
* not be called.
@@ -1375,9 +1334,6 @@
{
int i;
- CSTD_UNUSED(flags);
- KBASE_DEBUG_ASSERT(kbdev);
-
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbase_pm_is_gpu_lost(kbdev)) {
/* GPU access has been removed, reset will be done by
@@ -1386,6 +1342,9 @@
return false;
}
#endif
+
+ if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)
+ kbase_instr_hwcnt_on_unrecoverable_error(kbdev);
if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_NOT_PENDING,
@@ -1428,18 +1387,17 @@
*/
void kbase_reset_gpu(struct kbase_device *kbdev)
{
- KBASE_DEBUG_ASSERT(kbdev);
-
/* Note this is an assert/atomic_set because it is a software issue for
* a race to be occurring here
*/
- KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
- KBASE_RESET_GPU_PREPARED);
+ if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED))
+ return;
atomic_set(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_COMMITTED);
- dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
- kbdev->reset_timeout_ms);
+ if (!kbase_is_quick_reset_enabled(kbdev))
+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
+ kbdev->reset_timeout_ms);
hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
@@ -1452,18 +1410,17 @@
void kbase_reset_gpu_locked(struct kbase_device *kbdev)
{
- KBASE_DEBUG_ASSERT(kbdev);
-
/* Note this is an assert/atomic_set because it is a software issue for
* a race to be occurring here
*/
- KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
- KBASE_RESET_GPU_PREPARED);
+ if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED))
+ return;
atomic_set(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_COMMITTED);
- dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
- kbdev->reset_timeout_ms);
+ if (!kbase_is_quick_reset_enabled(kbdev))
+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
+ kbdev->reset_timeout_ms);
hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
HRTIMER_MODE_REL);
@@ -1497,6 +1454,11 @@
return false;
return true;
+}
+
+bool kbase_reset_gpu_is_not_pending(struct kbase_device *kbdev)
+{
+ return atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING;
}
int kbase_reset_gpu_wait(struct kbase_device *kbdev)
@@ -1540,9 +1502,9 @@
#ifdef CONFIG_MALI_BIFROST_DEBUG
dev_dbg(kbdev->dev,
"Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK from 0x%lx to 0x%lx (mask is 0x%lx)\n",
- (unsigned long int)affinity,
- (unsigned long int)result,
- (unsigned long int)limited_core_mask);
+ (unsigned long)affinity,
+ (unsigned long)result,
+ (unsigned long)limited_core_mask);
#else
CSTD_UNUSED(kbdev);
#endif
--
Gitblit v1.6.2