// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
|
/*
|
*
|
* (C) COPYRIGHT 2014-2022 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
/*
|
* GPU backend instrumentation APIs.
|
*/
|
|
#include <mali_kbase.h>
|
#include <gpu/mali_kbase_gpu_regmap.h>
|
#include <mali_kbase_hwaccess_instr.h>
|
#include <device/mali_kbase_device.h>
|
#include <backend/gpu/mali_kbase_instr_internal.h>
|
|
static int wait_prfcnt_ready(struct kbase_device *kbdev)
|
{
|
u32 loops;
|
|
for (loops = 0; loops < KBASE_PRFCNT_ACTIVE_MAX_LOOPS; loops++) {
|
const u32 prfcnt_active = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
|
GPU_STATUS_PRFCNT_ACTIVE;
|
if (!prfcnt_active)
|
return 0;
|
}
|
|
dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n");
|
return -EBUSY;
|
}
|
|
int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
|
struct kbase_context *kctx,
|
struct kbase_instr_hwcnt_enable *enable)
|
{
|
unsigned long flags;
|
int err = -EINVAL;
|
u32 irq_mask;
|
u32 prfcnt_config;
|
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
|
/* alignment failure */
|
if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1)))
|
return err;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) {
|
/* Instrumentation is already enabled */
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
return err;
|
}
|
|
if (kbase_is_gpu_removed(kbdev)) {
|
/* GPU has been removed by Arbiter */
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
return err;
|
}
|
|
/* Enable interrupt */
|
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask |
|
PRFCNT_SAMPLE_COMPLETED);
|
|
/* In use, this context is the owner */
|
kbdev->hwcnt.kctx = kctx;
|
/* Remember the dump address so we can reprogram it later */
|
kbdev->hwcnt.addr = enable->dump_buffer;
|
kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes;
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
|
/* Configure */
|
prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT;
|
#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
|
prfcnt_config |= kbdev->hwcnt.backend.override_counter_set
|
<< PRFCNT_CONFIG_SETSELECT_SHIFT;
|
#else
|
prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT;
|
#endif
|
|
/* Wait until prfcnt config register can be written */
|
err = wait_prfcnt_ready(kbdev);
|
if (err)
|
return err;
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
|
prfcnt_config | PRFCNT_CONFIG_MODE_OFF);
|
|
/* Wait until prfcnt is disabled before writing configuration registers */
|
err = wait_prfcnt_ready(kbdev);
|
if (err)
|
return err;
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
|
enable->dump_buffer & 0xFFFFFFFF);
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
|
enable->dump_buffer >> 32);
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
|
enable->fe_bm);
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
|
enable->shader_bm);
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
|
enable->mmu_l2_bm);
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
|
enable->tiler_bm);
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
|
prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL);
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
|
kbdev->hwcnt.backend.triggered = 1;
|
wake_up(&kbdev->hwcnt.backend.wait);
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
|
dev_dbg(kbdev->dev, "HW counters dumping set-up for context %pK", kctx);
|
return 0;
|
}
|
|
static void kbasep_instr_hwc_disable_hw_prfcnt(struct kbase_device *kbdev)
|
{
|
u32 irq_mask;
|
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
lockdep_assert_held(&kbdev->hwcnt.lock);
|
|
if (kbase_is_gpu_removed(kbdev))
|
/* GPU has been removed by Arbiter */
|
return;
|
|
/* Disable interrupt */
|
irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask & ~PRFCNT_SAMPLE_COMPLETED);
|
|
/* Wait until prfcnt config register can be written, then disable the counters.
|
* Return value is ignored as we are disabling anyway.
|
*/
|
wait_prfcnt_ready(kbdev);
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0);
|
|
kbdev->hwcnt.kctx = NULL;
|
kbdev->hwcnt.addr = 0ULL;
|
kbdev->hwcnt.addr_bytes = 0ULL;
|
}
|
|
int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx)
|
{
|
unsigned long flags, pm_flags;
|
struct kbase_device *kbdev = kctx->kbdev;
|
|
while (1) {
|
spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags);
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
|
/* Instrumentation is in unrecoverable error state,
|
* there is nothing for us to do.
|
*/
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
|
/* Already disabled, return no error. */
|
return 0;
|
}
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) {
|
/* Instrumentation is not enabled */
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
|
return -EINVAL;
|
}
|
|
if (kbdev->hwcnt.kctx != kctx) {
|
/* Instrumentation has been setup for another context */
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
|
return -EINVAL;
|
}
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE)
|
break;
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
|
|
/* Ongoing dump/setup - wait for its completion */
|
wait_event(kbdev->hwcnt.backend.wait,
|
kbdev->hwcnt.backend.triggered != 0);
|
}
|
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
|
kbdev->hwcnt.backend.triggered = 0;
|
|
kbasep_instr_hwc_disable_hw_prfcnt(kbdev);
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags);
|
|
dev_dbg(kbdev->dev, "HW counters dumping disabled for context %pK",
|
kctx);
|
|
return 0;
|
}
|
|
int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
|
{
|
unsigned long flags;
|
int err = -EINVAL;
|
struct kbase_device *kbdev = kctx->kbdev;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
if (kbdev->hwcnt.kctx != kctx) {
|
/* The instrumentation has been setup for another context */
|
goto unlock;
|
}
|
|
if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) {
|
/* HW counters are disabled or another dump is ongoing, or we're
|
* resetting, or we are in unrecoverable error state.
|
*/
|
goto unlock;
|
}
|
|
if (kbase_is_gpu_removed(kbdev)) {
|
/* GPU has been removed by Arbiter */
|
goto unlock;
|
}
|
|
kbdev->hwcnt.backend.triggered = 0;
|
|
/* Mark that we're dumping - the PF handler can signal that we faulted
|
*/
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING;
|
|
/* Wait until prfcnt is ready to request dump */
|
err = wait_prfcnt_ready(kbdev);
|
if (err)
|
goto unlock;
|
|
/* Reconfigure the dump address */
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
|
kbdev->hwcnt.addr & 0xFFFFFFFF);
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
|
kbdev->hwcnt.addr >> 32);
|
|
/* Start dumping */
|
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL,
|
kbdev->hwcnt.addr);
|
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
|
GPU_COMMAND_PRFCNT_SAMPLE);
|
|
dev_dbg(kbdev->dev, "HW counters dumping done for context %pK", kctx);
|
|
unlock:
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
return err;
|
}
|
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump);
|
|
bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx,
|
bool * const success)
|
{
|
unsigned long flags;
|
bool complete = false;
|
struct kbase_device *kbdev = kctx->kbdev;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) {
|
*success = true;
|
complete = true;
|
} else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
|
*success = false;
|
complete = true;
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
|
}
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
|
return complete;
|
}
|
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete);
|
|
void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
|
{
|
unsigned long flags;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
/* If the state is in unrecoverable error, we already wake_up the waiter
|
* and don't need to do any action when sample is done.
|
*/
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
|
kbdev->hwcnt.backend.triggered = 1;
|
wake_up(&kbdev->hwcnt.backend.wait);
|
} else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) {
|
/* All finished and idle */
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
|
kbdev->hwcnt.backend.triggered = 1;
|
wake_up(&kbdev->hwcnt.backend.wait);
|
}
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
}
|
|
int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx)
|
{
|
struct kbase_device *kbdev = kctx->kbdev;
|
unsigned long flags;
|
int err;
|
|
/* Wait for dump & cache clean to complete */
|
wait_event(kbdev->hwcnt.backend.wait,
|
kbdev->hwcnt.backend.triggered != 0);
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) {
|
err = -EINVAL;
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
|
} else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
|
err = -EIO;
|
} else {
|
/* Dump done */
|
KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state ==
|
KBASE_INSTR_STATE_IDLE);
|
err = 0;
|
}
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
|
return err;
|
}
|
|
int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
|
{
|
unsigned long flags;
|
int err = -EINVAL;
|
struct kbase_device *kbdev = kctx->kbdev;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
/* Check it's the context previously set up and we're not in IDLE
|
* state.
|
*/
|
if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state !=
|
KBASE_INSTR_STATE_IDLE)
|
goto unlock;
|
|
if (kbase_is_gpu_removed(kbdev)) {
|
/* GPU has been removed by Arbiter */
|
goto unlock;
|
}
|
|
/* Wait until prfcnt is ready to clear */
|
err = wait_prfcnt_ready(kbdev);
|
if (err)
|
goto unlock;
|
|
/* Clear the counters */
|
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0);
|
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
|
GPU_COMMAND_PRFCNT_CLEAR);
|
|
unlock:
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
return err;
|
}
|
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear);
|
|
void kbase_instr_hwcnt_on_unrecoverable_error(struct kbase_device *kbdev)
|
{
|
unsigned long flags;
|
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
/* If we already in unrecoverable error state, early return. */
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) {
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
return;
|
}
|
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR;
|
|
/* Need to disable HW if it's not disabled yet. */
|
if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED)
|
kbasep_instr_hwc_disable_hw_prfcnt(kbdev);
|
|
/* Wake up any waiters. */
|
kbdev->hwcnt.backend.triggered = 1;
|
wake_up(&kbdev->hwcnt.backend.wait);
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
}
|
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_unrecoverable_error);
|
|
void kbase_instr_hwcnt_on_before_reset(struct kbase_device *kbdev)
|
{
|
unsigned long flags;
|
|
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
|
|
/* A reset is the only way to exit the unrecoverable error state */
|
if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR)
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
|
|
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
|
}
|
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_before_reset);
|
|
int kbase_instr_backend_init(struct kbase_device *kbdev)
|
{
|
spin_lock_init(&kbdev->hwcnt.lock);
|
|
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED;
|
|
init_waitqueue_head(&kbdev->hwcnt.backend.wait);
|
|
kbdev->hwcnt.backend.triggered = 0;
|
|
#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
|
/* Use the build time option for the override default. */
|
#if defined(CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY)
|
kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_SECONDARY;
|
#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY)
|
kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_TERTIARY;
|
#else
|
/* Default to primary */
|
kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_PRIMARY;
|
#endif
|
#endif
|
return 0;
|
}
|
|
void kbase_instr_backend_term(struct kbase_device *kbdev)
|
{
|
CSTD_UNUSED(kbdev);
|
}
|
|
#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS
|
void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev)
|
{
|
/* No validation is done on the debugfs input. Invalid input could cause
|
* performance counter errors. This is acceptable since this is a debug
|
* only feature and users should know what they are doing.
|
*
|
* Valid inputs are the values accepted bythe SET_SELECT bits of the
|
* PRFCNT_CONFIG register as defined in the architecture specification.
|
*/
|
debugfs_create_u8("hwcnt_set_select", 0644,
|
kbdev->mali_debugfs_directory,
|
(u8 *)&kbdev->hwcnt.backend.override_counter_set);
|
}
|
#endif
|
